diff --git a/statvar_imports/us_bls/cpi_category/cpi_category_download.py b/statvar_imports/us_bls/cpi_category/cpi_category_download.py index c282c9a8c0..1ab76a8f43 100644 --- a/statvar_imports/us_bls/cpi_category/cpi_category_download.py +++ b/statvar_imports/us_bls/cpi_category/cpi_category_download.py @@ -404,7 +404,7 @@ def main(): } logging.info("\n--- Starting download process for yearly ZIP archives ---") - years_to_download_zip = list(range(2010, 2024)) # Covers 2010 up to and including 2023 + years_to_download_zip = list(range(2010, current_year)) # Covers 2010 up to and including previous year for year in years_to_download_zip: zip_url = generate_zip_url(year) @@ -427,6 +427,22 @@ def main(): logging.fatal(f"CRITICAL ERROR: Unzipping or processing of {filename} failed. Cannot proceed.") else: logging.info(f"Successfully processed ZIP for Year {year}.") + else: + logging.info(f"ZIP archive for {year} not found. Downloading all monthly files for cpi-u and cpi-w.") + for month in range(1, 13): + month_str = f"{month:02d}" + for category_prefix in ["cpi-u", "cpi-w"]: + url = f"https://www.bls.gov/cpi/tables/supplemental-files/{category_prefix}-{year}{month_str}.xlsx" + filename = url.split('/')[-1] + if category_prefix == "cpi-u": + file_save_path = os.path.join(cpi_u_folder, filename) + else: + file_save_path = os.path.join(cpi_w_folder, filename) + + try: + download_file(url, file_save_path, timeout=args.timeout) + except Exception as e: + logging.warning(f"Failed to download monthly file {url} for year {year} (fallback): {e}") logging.info("\n--- Listing files in respective folders after all downloads/extractions ---") logging.info("Files in 'cpi-u' folder:") diff --git a/statvar_imports/us_bls/cpi_category/cpi_u_pvmap.csv b/statvar_imports/us_bls/cpi_category/cpi_u_pvmap.csv index 481366e8b2..ab5eb4d27a 100644 --- a/statvar_imports/us_bls/cpi_category/cpi_u_pvmap.csv +++ b/statvar_imports/us_bls/cpi_category/cpi_u_pvmap.csv @@ -219,6 +219,7 @@ Candy and chewing gum(5),consumerGoodsCategory,CandyAndChewingGum,unit,IndexPoin Candy and chewing gum(5)(6),consumerGoodsCategory,CandyAndChewingGum,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Canned fruits and vegetables(5),consumerGoodsCategory,CannedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Canned fruits and vegetables(6),consumerGoodsCategory,CannedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, +Canned fruits(4)(5)(6),consumerGoodsCategory,CannedFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Canned fruits(5)(6),consumerGoodsCategory,CannedFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Canned fruits(6)(7),consumerGoodsCategory,CannedFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Canned vegetables(5)(6),consumerGoodsCategory,CannedVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, @@ -284,6 +285,7 @@ Delivery services(4)(5),consumerGoodsCategory,DeliveryServices,unit,IndexPointBa Delivery services(5),consumerGoodsCategory,DeliveryServices,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Delivery services(6),consumerGoodsCategory,DeliveryServices,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Dental services,consumerGoodsCategory,DentalServices,,,,,,,,,, +Dental services(4),consumerGoodsCategory,DentalServices,,,,,,,,,, Dental services(11),consumerGoodsCategory,DentalServices,,,,,,,,,, Dishes and flatware(4)(5),consumerGoodsCategory,DishesAndFlatware,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Dishes and flatware(5)(6),consumerGoodsCategory,DishesAndFlatware,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, @@ -621,6 +623,7 @@ Nonprescription drugs(11),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexP Nonprescription drugs(4)(11),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,, Nonprescription drugs(4)(12),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,, Nonprescription drugs(5)(12),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,, +Nursing homes and adult day services(4)(17),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,, Nursing homes and adult day services(11)(17),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,, Nursing homes and adult day services(16),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,, Nursing homes and adult day services(17),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,, @@ -647,6 +650,7 @@ Other fats and oils including peanut butter(6),consumerGoodsCategory,OtherFatsAn Other food at home,consumerGoodsCategory,OtherFoodAtHome,,,,,,,,,, Other food away from home(4)(5),consumerGoodsCategory,OtherFoodAwayFromHome,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other food away from home(5)(6),consumerGoodsCategory,OtherFoodAwayFromHome,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, +Other food away from home(5),consumerGoodsCategory,OtherFoodAwayFromHome,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other foods,consumerGoodsCategory,OtherFoods,,,,,,,,,, Other fresh fruits(5),consumerGoodsCategory,OtherFreshFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other fresh fruits(6),consumerGoodsCategory,OtherFreshFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, @@ -678,6 +682,7 @@ Other personal services(4)(12),consumerGoodsCategory,OtherPersonalServices,unit, Other personal services(5)(12),consumerGoodsCategory,OtherPersonalServices,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,, Other pork including roasts and picnics(5),consumerGoodsCategory,OtherPorkIncludingRoastAndPicnic,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, "Other pork including roasts, steaks, and ribs(5)",consumerGoodsCategory,OtherPorkIncludingRoastsSteaksAndRibs,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, +"Other pork including roasts, steaks, and ribs(4)(5)",consumerGoodsCategory,OtherPorkIncludingRoastsSteaksAndRibs,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, "Other pork including roasts, steaks, and ribs(6)",consumerGoodsCategory,OtherPorkIncludingRoastsSteaksAndRibs,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other poultry including turkey(5),consumerGoodsCategory,OtherPoultryIncludingTurkey,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other processed fruits and vegetables including dried(5),consumerGoodsCategory,OtherProcessedFruitsAndVegetablesIncludingDried,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, @@ -687,6 +692,7 @@ Other recreation services(6),consumerGoodsCategory,OtherRecreationServices,unit, Other recreational goods(5),consumerGoodsCategory,OtherRecreationalGoods,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other recreational goods(6),consumerGoodsCategory,OtherRecreationalGoods,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other services,consumerGoodsCategory,OtherServices,,,,,,,,,, +Other sweets(4)(5),consumerGoodsCategory,OtherSweets,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other sweets(5),consumerGoodsCategory,OtherSweets,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other sweets(6),consumerGoodsCategory,OtherSweets,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Other uncooked poultry including turkey(5),consumerGoodsCategory,OtherUncookedPoultryIncludingTurkey,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, @@ -717,6 +723,7 @@ Parking fees and tolls(6)(7),consumerGoodsCategory,ParkingFeesAndTolls,unit,Inde Peanut butter(4)(5)(6),consumerGoodsCategory,PeanutButter,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Peanut butter(5)(6)(7),consumerGoodsCategory,PeanutButter,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Personal care,consumerGoodsCategory,PersonalCare,,,,,,,,,, +Personal care products,consumerGoodsCategory,PersonalCareProducts,,,,,,,,,, Personal care products(4),consumerGoodsCategory,PersonalCareProducts,,,,,,,,,, Personal care products(5),consumerGoodsCategory,PersonalCareProducts,,,,,,,,,, Personal care services(4),consumerGoodsCategory,PersonalCareServices,,,,,,,,,, @@ -745,6 +752,7 @@ Photographers and film processing(4)(5),consumerGoodsCategory,PhotographerAndFil Photographers and photo processing(4)(5),consumerGoodsCategory,PhotographersAndPhotoProcessing,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Photographers and photo processing(5)(6),consumerGoodsCategory,PhotographersAndPhotoProcessing,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Photographic equipment and supplies,consumerGoodsCategory,PhotographicEquipmentAndSupplies,,,,,,,,,, +Photographic equipment and supplies(4),consumerGoodsCategory,PhotographicEquipmentAndSupplies,,,,,,,,,, Photographic equipment(4)(5)(6),consumerGoodsCategory,PhotographicEquipment,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Photographic equipment(5)(6),consumerGoodsCategory,PhotographicEquipment,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Photographic equipment(6)(7),consumerGoodsCategory,PhotographicEquipment,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, @@ -778,6 +786,7 @@ Processed fish and seafood(6),consumerGoodsCategory,ProcessedFishAndSeafood,unit Processed fruits and vegetables(5),consumerGoodsCategory,ProcessedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Processed fruits and vegetables(6),consumerGoodsCategory,ProcessedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Professional services,consumerGoodsCategory,ProfessionalServices,,,,,,,,,, +Professional services(4),consumerGoodsCategory,ProfessionalServices,,,,,,,,,, "Propane, kerosene, and firewood(10)",consumerGoodsCategory,PropaneKeroseneAndFirewood,unit,IndexPointBasePeriodDecember1986Equals100,,,,,,,, "Propane, kerosene, and firewood(4)(9)",consumerGoodsCategory,PropaneKeroseneAndFirewood,unit,IndexPointBasePeriodDecember1986Equals100,,,,,,,, "Propane, kerosene, and firewood(9)",consumerGoodsCategory,PropaneKeroseneAndFirewood,unit,IndexPointBasePeriodDecember1986Equals100,,,,,,,, @@ -908,6 +917,7 @@ Uncooked beef roasts(5)(6),consumerGoodsCategory,UncookedBeefRoasts,unit,IndexPo Uncooked beef steaks(4)(5),consumerGoodsCategory,UncookedBeefSteaks,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Uncooked beef steaks(5),consumerGoodsCategory,UncookedBeefSteaks,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, Uncooked beef steaks(6),consumerGoodsCategory,UncookedBeefSteaks,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,, +Uncooked ground beef,consumerGoodsCategory,UncookedGroundBeef,,,,,,,,,, Uncooked ground beef(4),consumerGoodsCategory,UncookedGroundBeef,,,,,,,,,, Uncooked ground beef(5),consumerGoodsCategory,UncookedGroundBeef,,,,,,,,,, Uncooked other beef and veal(4)(5),consumerGoodsCategory,UncookedOtherBeefAndVeal,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,