import pymysql import time import requests from w3lib.html import remove_tags from bs4 import BeautifulSoup condition=True while condition: import datetime # 获取时间 now_time = datetime.datetime.now() next_time = now_time + datetime.timedelta(days=+1) next_year = next_time.date().year next_month = next_time.date().month next_day = next_time.date().day next_time = datetime.datetime.strptime(str(next_year) + "-" + str(next_month) + "-" + str(next_day) + " 17:00:01","%Y-%m-%d %H:%M:%S") timer_start_time = (next_time - now_time).total_seconds() page =requests.get("https://www.federalreserve.gov/releases/h6/current/default.htm") page=page.text soup = BeautifulSoup(page, 'html.parser') page_tbody = soup.find_all('tbody') # 获取三个表数据MoneyStockMeasures，SeasonallyAdjusted，NotSeasonallyAdjusted MoneyStockMeasures = page_tbody[0] SeasonallyAdjusted = page_tbody[1] NotSeasonallyAdjusted = page_tbody[2] MoneyStockMeasures = remove_tags(str(MoneyStockMeasures)) SeasonallyAdjusted = remove_tags(str(SeasonallyAdjusted)) NotSeasonallyAdjusted = remove_tags(str(NotSeasonallyAdjusted)) # 修改时间字段 MoneyStockMeasures=MoneyStockMeasures.replace('Jan.', '1') MoneyStockMeasures = MoneyStockMeasures.replace('Feb.', '2') MoneyStockMeasures = MoneyStockMeasures.replace('Mar.', '3') MoneyStockMeasures = MoneyStockMeasures.replace('Apr.', '4') MoneyStockMeasures = MoneyStockMeasures.replace('May', '5') MoneyStockMeasures = MoneyStockMeasures.replace('June', '6') MoneyStockMeasures = MoneyStockMeasures.replace('July', '7') MoneyStockMeasures = MoneyStockMeasures.replace('Aug.', '8') MoneyStockMeasures = MoneyStockMeasures.replace('Sept.', '9') MoneyStockMeasures = MoneyStockMeasures.replace('Oct.', '10') MoneyStockMeasures = MoneyStockMeasures.replace('Nov.', '11') MoneyStockMeasures = MoneyStockMeasures.replace('Dec.', '12') MoneyStockMeasures = MoneyStockMeasures.replace('e', '') MoneyStockMeasures = MoneyStockMeasures.split() SeasonallyAdjusted = SeasonallyAdjusted.replace('Jan.', '1') SeasonallyAdjusted = SeasonallyAdjusted.replace('Feb.', '2') SeasonallyAdjusted = SeasonallyAdjusted.replace('Mar.', '3') SeasonallyAdjusted = SeasonallyAdjusted.replace('Apr.', '4') SeasonallyAdjusted = SeasonallyAdjusted.replace('May', '5') SeasonallyAdjusted = SeasonallyAdjusted.replace('June', '6') SeasonallyAdjusted = SeasonallyAdjusted.replace('July', '7') SeasonallyAdjusted = SeasonallyAdjusted.replace('Aug.', '8') SeasonallyAdjusted = SeasonallyAdjusted.replace('Sept.', '9') SeasonallyAdjusted = SeasonallyAdjusted.replace('Oct.', '10') SeasonallyAdjusted = SeasonallyAdjusted.replace('Nov.', '11') SeasonallyAdjusted = SeasonallyAdjusted.replace('Dec.', '12') SeasonallyAdjusted = SeasonallyAdjusted.replace('e', '') SeasonallyAdjusted = SeasonallyAdjusted.split() NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('Jan.', '1') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('Feb.', '2') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('Mar.', '3') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('Apr.', '4') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('May', '5') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('June', '6') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('July', '7') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('Aug.', '8') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('Sept.', '9') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('Oct.', '10') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('Nov.', '11') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('Dec.', '12') NotSeasonallyAdjusted = NotSeasonallyAdjusted.replace('e', '') NotSeasonallyAdjusted = NotSeasonallyAdjusted.split() condition1=0 for i in range(17): date1= MoneyStockMeasures[condition1 + 1] + '/' + MoneyStockMeasures[condition1 + 0] + '/1' format = '%Y/%m/%d' from datetime import datetime # 获取数据时间和各项数据的值 date1 = datetime.strptime(date1, format) value11=MoneyStockMeasures[condition1 + 2] value11 = value11.replace(",", "") value12=MoneyStockMeasures[condition1 + 3] value12 = value12.replace(",", "") value13=MoneyStockMeasures[condition1 + 7] value13 = value13.replace(",", "") value14=MoneyStockMeasures[condition1 + 8] value14 = value14.replace(",", "") value15=MoneyStockMeasures[condition1 + 4] value15 = value15.replace(",", "") value16=MoneyStockMeasures[condition1 + 5] value16 = value16.replace(",", "") value17=MoneyStockMeasures[condition1 + 6] value17 = value17.replace(",", "") value18=MoneyStockMeasures[condition1 + 9] value18 = value18.replace(",", "") value19=MoneyStockMeasures[condition1 + 10] value19 = value19.replace(",", "") value20=MoneyStockMeasures[condition1 + 11] value20 = value20.replace(",", "") condition1+=12 db = pymysql.connect(host="127.0.0.1",user="root",password="2GS@bPYcgiMyL14A",database="Macroeconomics",port=4423) cursor = db.cursor() sql = "select date from MoneyStockMeasures order by date desc limit 1" cursor.execute(sql) old_date = cursor.fetchall() date2=old_date[0][0] if i != 16: sql="UPDATE MoneyStockMeasures SET adjustedM1=%s,adjustedM2=%s,notAdjustedM1=%s,notAdjustedM2=%s,currencyincirculation=%s,reserveBalances=%s,monetaryBase=%s,totalReserves=%s,totalMborrowings_M=%s,nonborrowedReserves=%s WHERE date= '%s'"%(value11, value12, value13, value14, value15, value16, value17, value18, value19, value20, date1) cursor.execute(sql) db.commit() db.close() else: if date1 == date2: sql = "UPDATE MoneyStockMeasures SET adjustedM1=%s,adjustedM2=%s,notAdjustedM1=%s,notAdjustedM2=%s,currencyincirculation=%s,reserveBalances=%s,monetaryBase=%s,totalReserves=%s,totalMborrowings_M=%s,nonborrowedReserves=%s WHERE date= '%s'" % ( value11, value12, value13, value14, value15, value16, value17, value18, value19, value20, date1) cursor.execute(sql) db.commit() db.close() else: sql = "insert into MoneyStockMeasures(date,adjustedM1,adjustedM2,notAdjustedM1,notAdjustedM2,currencyincirculation,reserveBalances,monetaryBase,totalReserves,totalMborrowings_M,nonborrowedReserves)values('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')" % ( date1, value11, value12, value13, value14, value15, value16, value17, value18, value19, value20) cursor.execute(sql) db.commit() db.close() condition2=0 for i in range(17): date3= SeasonallyAdjusted[condition2 + 1] + '/' + SeasonallyAdjusted[condition2 + 0] + '/1' format = '%Y/%m/%d' from datetime import datetime date3 = datetime.strptime(date3, format) value21=SeasonallyAdjusted[condition2 + 2] value21 = value21.replace(",", "") value22=SeasonallyAdjusted[condition2 + 3] value22 = value22.replace(",", "") value23=SeasonallyAdjusted[condition2 + 4] value23 = value23.replace(",", "") value24=SeasonallyAdjusted[condition2 + 5] value24 = value24.replace(",", "") value25=SeasonallyAdjusted[condition2 + 6] value25 = value25.replace(",", "") condition2+=7 db = pymysql.connect(host="127.0.0.1",user="root",password="2GS@bPYcgiMyL14A",database="Macroeconomics",port=4423) cursor = db.cursor() sql = "select date from SeasonallyAdjusted order by date desc limit 1" cursor.execute(sql) old_date = cursor.fetchall() date4=old_date[0][0] if i != 16: sql="UPDATE SeasonallyAdjusted SET currencyM1=%s,demandM1=%s,otherLiquid=%s,smallDenominationTimeNonM1M2=%s,retailMoneyMarketFundsNonM1M2=%s WHERE date= '%s'"%(value21, value22, value23, value24, value25, date3) cursor.execute(sql) db.commit() db.close() else: if date3 == date4: sql = "UPDATE SeasonallyAdjusted SET currencyM1=%s,demandM1=%s,otherLiquid=%s,smallDenominationTimeNonM1M2=%s,retailMoneyMarketFundsNonM1M2=%s WHERE date= '%s'" % ( value21, value22, value23, value24, value25, date3) cursor.execute(sql) db.commit() db.close() else: sql = "insert into SeasonallyAdjusted(date,currencyM1,demandM1,otherLiquid,smallDenominationTimeNonM1M2,retailMoneyMarketFundsNonM1M2)values('%s','%s','%s','%s','%s','%s')" % ( date3, value21, value22, value23, value24, value25) cursor.execute(sql) db.commit() db.close() condition3=0 for i in range(17): date5= NotSeasonallyAdjusted[condition3 + 1] + '/' + NotSeasonallyAdjusted[condition3 + 0] + '/1' format = '%Y/%m/%d' from datetime import datetime date5 = datetime.strptime(date5, format) value31=NotSeasonallyAdjusted[condition3 + 2] value31 = value31.replace(",", "") value32=NotSeasonallyAdjusted[condition3 + 3] value32 = value32.replace(",", "") value33=NotSeasonallyAdjusted[condition3 + 4] value33 = value33.replace(",", "") value34=NotSeasonallyAdjusted[condition3 + 5] value34 = value34.replace(",", "") value35=NotSeasonallyAdjusted[condition3 + 6] value35 = value35.replace(",", "") value36 = NotSeasonallyAdjusted[condition3 + 7] value36 = value36.replace(",", "") value37 = NotSeasonallyAdjusted[condition3 + 8] value37 = value37.replace(",", "") value38 = NotSeasonallyAdjusted[condition3 + 9] value38 = value38.replace(",", "") condition3+=10 db = pymysql.connect(host="127.0.0.1",user="root",password="2GS@bPYcgiMyL14A",database="Macroeconomics",port=4423) cursor = db.cursor() sql = "select date from NotSeasonallyAdjusted order by date desc limit 1" cursor.execute(sql) old_date = cursor.fetchall() date6=old_date[0][0] if i != 16: sql="UPDATE NotSeasonallyAdjusted SET currencyM1=%s,demandM1=%s,otherLiquid=%s,smallDenominationTimeNonM1M2=%s,retailMoneyMarketFundsNonM1M2=%s,atDepositoryInstitutions=%s,atMoneyMarketFunds=%s,total=%s WHERE date= '%s'"%(value31, value32, value33, value34, value35, value36, value37, value38, date5) cursor.execute(sql) db.commit() db.close() else: if date5 == date6: sql = "UPDATE NotSeasonallyAdjusted SET currencyM1=%s,demandM1=%s,otherLiquid=%s,smallDenominationTimeNonM1M2=%s,retailMoneyMarketFundsNonM1M2=%s,atDepositoryInstitutions=%s,atMoneyMarketFunds=%s,total=%s WHERE date= '%s'" % ( value31, value32, value33, value34, value35, value36, value37, value38, date5) cursor.execute(sql) db.commit() db.close() else: sql = "insert into NotSeasonallyAdjusted(date,currencyM1,demandM1,otherLiquid,smallDenominationTimeNonM1M2,retailMoneyMarketFundsNonM1M2,atDepositoryInstitutions,atMoneyMarketFunds,total)values('%s','%s','%s','%s','%s','%s','%s','%s','%s')" % ( date5, value31, value32, value33, value34, value35, value36, value37, value38) cursor.execute(sql) db.commit() db.close() time.sleep(timer_start_time)