diff --git a/coinbus/Macroeconomic_FARBODI.py b/coinbus/Macroeconomic_FARBODI.py new file mode 100644 index 0000000..9af1333 --- /dev/null +++ b/coinbus/Macroeconomic_FARBODI.py @@ -0,0 +1,293 @@ +import time +import pymysql +import requests +from bs4 import BeautifulSoup +from w3lib.html import remove_tags +import datetime +while True: + try: + # now_time = datetime.datetime.now() + # next_time = now_time + datetime.timedelta(days=+1) + # next_year = next_time.date().year + # next_month = next_time.date().month + # next_day = next_time.date().day + # next_time = datetime.datetime.strptime(str(next_year) + "-" + str(next_month) + "-" + str(next_day) + " 20:45:01","%Y-%m-%d %H:%M:%S") + # timer_start_time = (next_time - now_time).total_seconds() + db = pymysql.connect(host="127.0.0.1",user="root",password="2GS@bPYcgiMyL14A",database="Macroeconomics",port=4423) + cursor = db.cursor() + page = requests.get("https://www.federalreserve.gov/releases/h41/current/default.htm") + page=page.text + page = BeautifulSoup(page, 'html.parser') + date = page.find_all('div', class_="dates") + # 获取数据 + date = remove_tags(str(date)) + # 删除多余字符 + date = date.replace("[", "") + date = date.replace("]", "") + date = date.replace("Release Date:", "") + date = date.replace(",","") + date = date.replace(" ","") + date = date.strip() + date = date.lstrip() + date1 = date[-4:] + date2 = date[-6:-4] + # 转换时间 + date = date.replace("January", "/01/") + date = date.replace("February", "/02/") + date = date.replace("March", "/03/") + date = date.replace("April", "/04/") + date = date.replace("May", "/05/") + date = date.replace("June", "/06/") + date = date.replace("July", "/07/") + date = date.replace("August", "/08/") + date = date.replace("September", "/09/") + date = date.replace("October", "/10/") + date = date.replace("November", "/11/") + date = date.replace("December", "/12/") + date = date1+date[0:4]+date2 + date_string = date + format = '%Y/%m/%d' + from datetime import datetime + date = datetime.strptime(date_string, format) + sql = "select time from CHAFRNFRAA order by id desc limit 1" + cursor.execute(sql) + db.commit() + old_time = cursor.fetchall() + ole_time=old_time[0][0] + except: + time.sleep(30) + continue + # 判断时间 + if date != ole_time: + page = page.find_all('span',style="font-family:'Courier New'; font-weight:bold") + page = remove_tags(str(page)) + page = page.replace(",", "") + page = page.replace("[", "") + page = page.replace("]", "") + page=page.split() + symbol= '' + list=[] + # 数据拼接 + for i in page: + if i =='-': + symbol= '-' + continue + if i =='+': + continue + if i =='(0)': + continue + if i =='...': + i='NULL' + value= symbol + i + symbol= '' + list+=[value] + sql = "insert into FARBODI(name,time,THIS_AVG_VALUE,CHANGE_LASTWEEK,CHANGE_LASTYEAR,THIS_VALUE)values(%s,%s,%s,%s,%s,%s)" + data_list=[('Reserve Bank credit',date,list[0],list[1],list[2],list[3]), + ('Securities held outright', date, list[4], list[5], list[6], list[7]), + ('U.S. Treasury securities', date, list[8], list[9], list[10], list[11]), + ('Bills', date, list[12], list[13], list[14], list[15]), + ('Notes and bonds, nominal', date, list[16], list[17], list[18], list[19]), + ('Notes and bonds, inflation-indexed', date, list[20], list[21], list[22], list[23]), + ('Inflation compensation', date, list[24], list[25], list[26], list[27]), + ('Federal agency debt securities', date, list[28], list[29], list[30], list[31]), + ('Mortgage-backed securities', date, list[32], list[33], list[34], list[35]), + ('Uposho', date, list[36], list[37], list[38], list[39]), + ('Udosho', date, list[40], list[41], list[42], list[43]), + ('Repurchase agreements', date, list[44], list[45], list[46], list[47]), + ('Foreign official', date, list[48], list[49], list[50], list[51]), + ('Others', date, list[52], list[53], list[54], list[55]), + ('Loans', date, list[56], list[57], list[58], list[59]), + ('Primary credit', date, list[60], list[61], list[62], list[63]), + ('Secondary credit', date, list[64], list[65], list[66], list[67]), + ('Seasonal credit', date, list[68], list[69], list[70], list[71]), + ('PPPLF', date, list[72], list[73], list[74], list[75]), + ('Bank Term Funding Program', date, list[76], list[77], list[78], list[79]), + ('Other credit extensions', date, list[80], list[81], list[82], list[83]), + ('NphoMFLLC(MSLP)', date, list[84], list[85], list[86], list[87]), + ('Net portfolio holdings of MLF LLC', date, list[88], list[89], list[90], list[91]), + ('Net portfolio holdings of TALF II LLC',date, list[92], list[93], list[94], list[95]), + ('Float',date, list[96], list[97], list[98], list[99]), + ('Central bank liquidity swaps',date, list[100], list[101], list[102], list[103]), + ('Other Federal Reserve assets',date, list[104], list[105], list[106], list[107]), + ('Foreign currency denominated assets',date, list[108], list[109], list[110], list[111]), + ('Gold stock',date, list[112], list[113], list[114], list[115]), + ('Special drawing rights certificate account',date, list[116], list[117], list[118], list[119]), + ('Treasury currency outstanding',date, list[120], list[121], list[122], list[123]), + ('Total factors supplying reserve funds',date, list[124], list[125], list[126], list[127])] + cursor.executemany(sql,data_list) + + + + + sql2 = "insert into FARBODIC(name,time,THIS_AVG_VALUE,CHANGE_LASTWEEK,CHANGE_LASTYEAR,THIS_VALUE)values(%s,%s,%s,%s,%s,%s)" + data_list2=[('Currency in circulation',date, list[128], list[129], list[130], list[131]), + ('Reverse repurchase agreements',date, list[132], list[133], list[134], list[135]), + ('Foreign official and international accounts',date, list[136], list[137], list[138], list[139]), + ('Others',date, list[140], list[141], list[142], list[143]), + ('Treasury cash holdings',date, list[144], list[145], list[146], list[147]), + ('DwFRBotrb',date, list[148], list[149], list[150], list[151]), + ('Tdhbdi',date, list[152], list[153], list[154], list[155]), + ('U.S. Treasury, General Account',date, list[156], list[157], list[158], list[159]), + ('Foreign official',date, list[160], list[161], list[162], list[163]), + ('Other',date, list[164], list[165], list[166], list[167]), + ('Treasury contributions to credit facilities',date, list[168], list[169], list[170], list[171]), + ('Other liabilities and capital',date, list[172], list[173], list[174], list[175]), + ('Tfotrbarf',date, list[176], list[177], list[178], list[179]), + ('RbwFRB',date, list[180], list[181], list[182], list[183])] + cursor.executemany(sql2,data_list2) + + + + sql3 = "insert into MI(name,time,THIS_AVG_VALUE,CHANGE_LASTWEEK,CHANGE_LASTYEAR,THIS_VALUE)values(%s,%s,%s,%s,%s,%s)" + data_list3=[('Shicffoaia',date, list[184], list[185], list[186], list[187]), + ('Marketable U.S. Treasury securities',date, list[188], list[189], list[190], list[191]), + ('Fadambs',date, list[192], list[193], list[194], list[195]), + ('Other securities',date, list[196], list[197], list[198], list[199]), + ('Securities lent to dealers',date, list[200], list[201], list[202], list[203]), + ('Overnight facility',date, list[204], list[205], list[206], list[207]), + ('U.S. Treasury securities',date, list[208], list[209], list[210], list[211]), + ('Federal agency debt securities',date,list[212], list[213], list[214], list[215])] + cursor.executemany(sql3,data_list3) + + + + sql4 = "insert into MDOSLASOAAL(name,time,D15,D16_D90,D91_Y1,Y1_Y5,Y5_Y10,Y10_,TOTAL)values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" + data_list4=[('Loans', date, list[216], list[217], list[218], list[219], list[220], list[221], list[222]), + ('USTsH', date, list[223], list[224], list[225], list[226], list[227], list[228], list[229]), + ('USTsWc', date, list[230], list[231], list[232], list[233], list[234], list[235], list[236]), + ('FadsH', date, list[237], list[238], list[239], list[240], list[241], list[242], list[243]), + ('FadsWc', date, list[244], list[245], list[246], list[247], list[248], list[249], list[250]), + ('MbsH', date, list[251], list[252], list[253], list[254], list[255], list[256], list[257]), + ('MbsWc', date, list[258], list[259], list[260], list[261], list[262], list[263], list[264]), + ('LphbMFLLC(MSLP)', date, list[265], list[266], list[267], list[268], list[269], list[270], list[271]), + ('Repurchase agreements', date, list[272], list[273], list[274], list[275], list[276], list[277], list[278]), + ('Central bank liquidity swaps', date, list[279], list[280], list[281], list[282], list[283], list[284], list[285]), + ('Reverse repurchase agreements', date, list[286], list[287], list[288], list[289], list[290], list[291], list[292]), + ('Term deposits', date, list[293], list[294], list[295], list[296], list[297], list[298], list[299])] + cursor.executemany(sql4,data_list4) + + + sql5 = "insert into SIOMS(name,time,value)values(%s,%s,%s)" + data_list5 = [('Mortgage-backed securities held outright', date, list[300]), + ('Residential mortgage-backed securities', date, list[301]), + ('Commercial mortgage-backed securities', date, list[302]), + ('Commitments to buy mortgage-backed securities', date, list[303]), + ('Commitments to sell mortgage-backed securities', date, list[304]), + ('Cash and cash equivalents', date, list[305])] + cursor.executemany(sql5, data_list5) + + + + sql6 = "insert into IOPAOCFL(name,time,OPAPTLLC,UPFAPT,FAAOA,TOTAL)values(%s,%s,%s,%s,%s,%s)" + data_list6 =[('MS Facilities LLC (Main Street Lending Program)',date, list[306], list[307], list[308], list[309])] + cursor.executemany(sql6, data_list6) + + + sql7 = "insert into CSOCOAFRB(name,time,EFC,THIS_VALUE,CHANGE_LASTWEEK,CHANGE_LASTYEAR)values(%s,%s,%s,%s,%s,%s)" + data_list7 = [('Gold certificate account', date, 'NULL', list[310], list[311], list[312]), + ('Special drawing rights certificate account', date, 'NULL', list[313], list[314], list[315]), + ('Coin', date, 'NULL', list[316], list[317], list[318]), + ('Supadraal', date, 'NULL', list[319], list[320], list[321]), + ('Securities held outright', date, 'NULL', list[322], list[323], list[324]), + ('U.S. Treasury securities', date, 'NULL', list[325], list[326], list[327]), + ('Bills', date, 'NULL', list[328], list[329], list[330]), + ('Notes and bonds, nominal', date, 'NULL', list[331], list[332], list[333]), + ('Notes and bonds, inflation-indexed', date, 'NULL', list[334], list[335], list[336]), + ('Inflation compensation', date, 'NULL', list[337], list[338], list[339]), + ('Federal agency debt securities', date, 'NULL', list[340], list[341], list[342]), + ('Mortgage-backed securities', date, 'NULL', list[343], list[344], list[345]), + ('Uposho', date, 'NULL', list[346], list[347], list[348]), + ('Udosho', date, 'NULL', list[349], list[350], list[351]), + ('Repurchase agreements', date, 'NULL', list[352], list[353], list[354]), + ('Loans', date, 'NULL', list[355], list[356], list[357]), + ('NphoMFLLC(MSLP)', date, 'NULL', list[358], list[359], list[360]), + ('NphoMLFLLC', date, 'NULL', list[361], list[362], list[363]), + ('Net portfolio holdings of TALF II LLC', date, 'NULL', list[364], list[365], list[366]), + ('Items in process of collection', date, 'NULL', list[367], list[368], list[369]), + ('Bank premises', date, 'NULL', list[370], list[371], list[372]), + ('Central bank liquidity swaps', date, 'NULL', list[373], list[374], list[375]), + ('Foreign currency denominated assets', date, 'NULL', list[376], list[377], list[378]), + ('Other assets', date, 'NULL', list[379], list[380], list[381]), + ('Total assets', date, 'NULL', list[382], list[383], list[384])] + cursor.executemany(sql7, data_list7) + + + + sql8 = "insert into CSOCOAFRBC(name,time,EFC,THIS_VALUE,CHANGE_LASTWEEK,CHANGE_LASTYEAR)values(%s,%s,%s,%s,%s,%s)" + data_list8 = [('FRnnoFBh', date, 'NULL', list[385], list[386], list[387]), + ('Reverse repurchase agreements', date, 'NULL', list[388], list[389], list[390]), + ('Deposits', date, 'NULL', list[391], list[392], list[393]), + ('Term deposits held by depository institutions', date, 'NULL', list[394], list[395], list[396]), + ('Other deposits held by depository institutions', date, 'NULL', list[397], list[398], list[399]), + ('U.S. Treasury, General Account', date, 'NULL', list[400], list[401], list[402]), + ('Foreign official', date, 'NULL', list[403], list[404], list[405]), + ('Other', date, 'NULL', list[406], list[407], list[408]), + ('Deferred availability cash items', date, 'NULL', list[409], list[410], list[411]), + ('Treasury contributions to credit facilities', date, 'NULL', list[412], list[413], list[414]), + ('Other liabilities and accrued dividends', date, 'NULL', list[415], list[416], list[417]), + ('Total liabilities', date, 'NULL', list[418], list[419], list[420]), + ('Capital paid in', date, 'NULL', list[421], list[422], list[423]), + ('Surplus', date, 'NULL', list[424], list[425], list[426]), + ('Other capital accounts', date, 'NULL', list[427], list[428], list[429]), + ('Total capital', date, 'NULL', list[430], list[431], list[432])] + cursor.executemany(sql8, data_list8) + + + + + + sql9 = "insert into SOCOEFRB(name,time,TOTAL,Boston,NewYork,Philadelphia,Cleveland,Richmond,Atlanta,Chicago,St_Louis,Minneapolis,Kansas_City,Dallas,San_Francisco)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" + data_list9=[('Gcasdrc', date, list[433], list[434], list[435], list[436], list[437], list[438], list[439], list[440], list[441], list[442], list[443], list[444], list[445]), + ('Coin', date, list[446], list[447], list[448], list[449], list[450], list[451], list[452], list[453], list[454], list[455], list[456], list[457], list[458]), + ('Supadraal', date, list[459], list[460], list[461], list[462], list[463], list[464], list[465], list[466], list[467], list[468], list[469], list[470], list[471]), + ('NphoMFLLC(MSLP)', date, list[472], list[473], list[474], list[475], list[476], list[477], list[478], list[479], list[480], list[481], list[482], list[483], list[484]), + ('Central bank liquidity swaps', date, list[485], list[486], list[487], list[488], list[489], list[490], list[491], list[492], list[493], list[494], list[495], list[496], list[497]), + ('Foreign currency denominated assets', date, list[498], list[499], list[500], list[501], list[502], list[503], list[504], list[505], list[506], list[507], list[508], list[509], list[510]), + ('Other assets', date, list[511], list[512], list[513], list[514], list[515], list[516], list[517], list[518], list[519], list[520], list[521], list[522], list[523]), + ('Interdistrict settlement account', date, list[524], list[525], list[526], list[527], list[528], list[529], list[530], list[531], list[532], list[533], list[534], list[535], list[536]), + ('Total assets', date, list[537], list[538], list[539], list[540], list[541], list[542], list[543], list[544], list[545], list[546], list[547], list[548], list[549])] + cursor.executemany(sql9, data_list9) + + + + + sql10 = "insert into SOCOEFRBC(name,time,TOTAL,Boston,NewYork,Philadelphia,Cleveland,Richmond,Atlanta,Chicago,St_Louis,Minneapolis,Kansas_City,Dallas,San_Francisco)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" + data_list10 = [('Federal Reserve notes, net', date, list[550], list[551], list[552], list[553], list[554], list[555], list[556], list[557], list[558], list[559], list[560], list[561], list[562]), + ('Reverse repurchase agreements', date, list[563], list[564], list[565], list[566], list[567], list[568], list[569], list[570], list[571], list[572], list[573], list[574], list[575]), + ('Deposits', date, list[576], list[577], list[578], list[579], list[580], list[581], list[582], list[583], list[584], list[585], list[586], list[587], list[588]), + ('Depository institutions', date, list[589], list[590], list[591], list[592], list[593], list[594], list[595], list[596], list[597], list[598], list[599], list[600], list[601]), + ('U.S. Treasury, General Account', date, list[602], list[603], list[604], list[605], list[606], list[607], list[608], list[609], list[610], list[611], list[612], list[613], list[614]), + ('Foreign official', date, list[615], list[616], list[617], list[618], list[619], list[620], list[621], list[622], list[623], list[624], list[625], list[626], list[627]), + ('Other', date, list[628], list[629], list[630], list[631], list[632], list[633], list[634], list[635], list[636], list[637], list[638], list[639], list[640]), + ('Earnings remittances due to the U.S. Treasury', date, list[641], list[642], list[643], list[644], list[645], list[646], list[647], list[648], list[649], list[650], list[651], list[652], list[653]), + ('Treasury contributions to credit facilities', date, list[654], list[655], list[656], list[657], list[658], list[659], list[660], list[661], list[662], list[663], list[664], list[665], list[666]), + ('Other liabilities and accrued dividends', date, list[667], list[668], list[669], list[670], list[671], list[672], list[673], list[674], list[675], list[676], list[677], list[678], list[679]), + ('Total liabilities', date, list[680], list[681], list[682], list[683], list[684], list[685], list[686], list[687], list[688], list[689], list[690], list[691], list[692]), + ('Capital paid in', date, list[693], list[694], list[695], list[696], list[697], list[698], list[699], list[700], list[701], list[702], list[703], list[704], list[705]), + ('Surplus', date, list[706], list[707], list[708], list[709], list[710], list[711], list[712], list[713], list[714], list[715], list[716], list[717], list[718]), + ('Other capital', date, list[719], list[720], list[721], list[722], list[723], list[724], list[725], list[726], list[727], list[728], list[729], list[730], list[731]), + ('Total liabilities and capital', date, list[732], list[733], list[734], list[735], list[736], list[737], list[738], list[739], list[740], list[741], list[742], list[743], list[744])] + cursor.executemany(sql10, data_list10) + + + + + + sql11 = "insert into CHAFRNFRAA(name,time,value)values(%s,%s,%s)" + data_list11 = [('Federal Reserve notes outstanding', date, list[745]), + ('LNhbFBnstc', date, list[746]), + ('Federal Reserve notes to be collateralized', date, list[747]), + ('Collateral held against Federal Reserve notes', date, list[748]), + ('Gold certificate account', date, list[749]), + ('Special drawing rights certificate account', date, list[750]), + ('UTadambsp', date, list[751]), + ('Other assets pledged', date, list[752]), + ('TUTadambs', date, list[753]), + ('LFvosurra', date, list[754]), + ('UTadambsetbp', date, list[755])] + cursor.executemany(sql11, data_list11) + db.commit() + else: + time.sleep(21600) + # time.sleep(timer_start_time) + diff --git a/coinbus/Macroeconomic_USTreasuriesSize.py b/coinbus/Macroeconomic_USTreasuriesSize.py new file mode 100644 index 0000000..e102e39 --- /dev/null +++ b/coinbus/Macroeconomic_USTreasuriesSize.py @@ -0,0 +1,85 @@ +import requests +import pymysql +from datetime import datetime +import time +old_transaction_mtd_amt=0 +condition=True +db = pymysql.connect(host="127.0.0.1",user="root",password="2GS@bPYcgiMyL14A",database="Macroeconomics",port=4423) +cursor = db.cursor() +while condition: + page = requests.get("https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v1/accounting/dts/public_debt_transactions?fields=record_date,transaction_type,security_type,transaction_mtd_amt&sort=-record_date") + page = page.json() + page = page['data'] + # 获取网站最新一条数据时间 + page_data = page[0:23] + for data in page_data: + sql = "select date from USTreasuriesSize order by date desc limit 1" + cursor.execute(sql) + old_date = cursor.fetchall() + old_date = str(old_date) + old_date = old_date[20:-11] + old_date = old_date.replace(' ', '') + old_date = old_date.replace(',', '/') + date_string = old_date + format = '%Y/%m/%d' + old_date = datetime.strptime(date_string, format) + + record_date = data['record_date'] + transaction_type = data['transaction_type'] + security_type=data['security_type'] + transaction_mtd_amt=data['transaction_mtd_amt'] + record_date = record_date.replace('-', '/') + format = '%Y/%m/%d' + record_date = datetime.strptime(record_date, format) + # 判断数据库最新一条数据时间和网站最新一条数据时间 + if record_date!=old_date: + sql = "insert into USTreasuriesSize(date)values('%s')" % (record_date) + cursor.execute(sql) + db.commit() + sql = "select id from USTreasuriesSize order by id desc limit 1" + cursor.execute(sql) + id = cursor.fetchall() + id = id[0][0] + for data in page_data: + transaction_type = data['transaction_type'] + security_type = data['security_type'] + transaction_mtd_amt = data['transaction_mtd_amt'] + if transaction_type == 'Issues': + if security_type == 'Bills': + transaction_mtd_amt= old_transaction_mtd_amt + int(transaction_mtd_amt) + sql1 = "update USTreasuriesSize set TBill=%s where id=%s" % (transaction_mtd_amt, id) + cursor.execute(sql1) + db.commit() + old_transaction_mtd_amt=transaction_mtd_amt + + elif security_type == 'Notes': + sql2 = "update USTreasuriesSize set TNote=%s where id=%s" % (transaction_mtd_amt, id) + cursor.execute(sql2) + db.commit() + + elif security_type == 'Bonds': + sql3 = "update USTreasuriesSize set TBond=%s where id=%s" % (transaction_mtd_amt, id) + cursor.execute(sql3) + db.commit() + + elif security_type == 'Inflation-Protected Securities Increment': + sql4 = "update USTreasuriesSize set TIPS=%s where id=%s" % (transaction_mtd_amt, id) + cursor.execute(sql4) + db.commit() + else: + continue + else: + continue + else: + continue + + old_transaction_mtd_amt=0 + time.sleep(21600) + + + + + + + + diff --git a/coinbus/Macroeconomic_USTreasuriesYields_v2.py b/coinbus/Macroeconomic_USTreasuriesYields_v2.py new file mode 100644 index 0000000..f99df46 --- /dev/null +++ b/coinbus/Macroeconomic_USTreasuriesYields_v2.py @@ -0,0 +1,96 @@ +import pymysql +import time +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.chrome.service import Service +from w3lib.html import remove_tags +from bs4 import BeautifulSoup +from datetime import datetime + +# 配置 Selenium +chrome_options = Options() +chrome_options.add_argument("--headless") +chrome_options.add_argument('--no-sandbox') +chrome_options.add_argument('--disable-gpu') +chrome_options.add_argument('--disable-dev-shm-usage') +chrome_options.add_argument('blink-settings=imagesEnabled=false') + +# 在 Selenium 3 中,直接指定 chrome_options 参数即可 +browser = webdriver.Chrome(executable_path="chromedriver", options=chrome_options) +# 将月份映射整理成一个字典 +MONTH_MAPPING = { + "Jan": "/1/", "Feb": "/2/", "Mar": "/3/", "Apr": "/4/", + "May": "/5/", "Jun": "/6/", "Jul": "/7/", "Aug": "/8/", + "Sep": "/9/", "Oct": "/10/", "Nov": "/11/", "Dec": "/12/" +} + +# 数据库连接配置 +DB_CONFIG = { + "host": "127.0.0.1", + "user": "root", + "password": "2GS@bPYcgiMyL14A", + "database": "Macroeconomics", + "port": 4423 +} + +def fetch_web_data(): + """抓取网页数据并解析日期和利率数据""" + browser.get("https://www.federalreserve.gov/releases/h15/") + soup = BeautifulSoup(browser.page_source, 'html.parser') + + # 获取日期 + date_text = soup.find_all('th', class_="colhead sticky sticky-row-cell")[-1].get_text(strip=True) + for month, replacement in MONTH_MAPPING.items(): + date_text = date_text.replace(month, replacement) + date = datetime.strptime(date_text.replace('*', ''), '%Y/%m/%d') + + # 获取利率数据 + data = [remove_tags(str(td)).strip() for td in soup.find_all('td', class_="data")] + return date, data + +def get_latest_db_date(): + """从数据库获取最新日期""" + with pymysql.connect(**DB_CONFIG) as conn: + with conn.cursor() as cursor: + cursor.execute("SELECT MAX(date) FROM USTreasuriesYields") + result = cursor.fetchone() + return result[0] if result[0] else None + +def insert_data(date, rates, mprime, dpcredit): + """插入数据到数据库""" + with pymysql.connect(**DB_CONFIG) as conn: + with conn.cursor() as cursor: + sql_treasuries = """ + INSERT INTO USTreasuriesYields (date, 1_Mo, 3_Mo, 6_Mo, 1_Yr, 2_Yr, 5_Yr, 10_Yr, 20_Yr, 30_Yr) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """ + sql_interest_rate = """ + INSERT INTO InterestRate (date, name, _value) + VALUES (%s, %s, %s) + """ + cursor.execute(sql_treasuries, [date] + rates) + cursor.execute(sql_interest_rate, (date, 'BPL', mprime)) + cursor.execute(sql_interest_rate, (date, 'DWPC', dpcredit)) + conn.commit() + +def main(): + while True: + new_date, data = fetch_web_data() + old_date = get_latest_db_date() + + if old_date and new_date <= old_date: + time.sleep(21600) # 6小时 + continue + + # 提取利率数据 + rates = [ + data[i].replace('ND', 'NULL') if i < len(data) else 'NULL' + for i in [104, 109, 114, 119, 124, 134, 144, 149, 154] + ] + mprime = data[54] if len(data) > 54 else 'NULL' + dpcredit = data[59] if len(data) > 59 else 'NULL' + + insert_data(new_date, rates, mprime, dpcredit) + +if __name__ == "__main__": + main() diff --git a/coinbus/Macroeconomic_WEI.py b/coinbus/Macroeconomic_WEI.py new file mode 100644 index 0000000..83995d1 --- /dev/null +++ b/coinbus/Macroeconomic_WEI.py @@ -0,0 +1,105 @@ +import time +from full_fred.fred import Fred +import pymysql +import time +fred=Fred('example_key.txt') +fred.set_api_key_file('example_key.txt') +times="00:00:00" +while True: + import datetime + # now_time = datetime.datetime.now() + # next_time = now_time + datetime.timedelta(days=+1) + # next_year = next_time.date().year + # next_month = next_time.date().month + # next_day = next_time.date().day + # next_time = datetime.datetime.strptime(str(next_year) + "-" + str(next_month) + "-" + str(next_day) + " 15:00:00","%Y-%m-%d %H:%M:%S") + # timer_start_time = (next_time - now_time).total_seconds() + data=fred.get_series_df('WEI') + data_date=data['date'] + data_value=data['value'] + # 获取网站数据 + list1=[] + list2=[] + for i in data_date: + list1+=[i] + for i2 in data_value: + list2+=[i2] + # 拿最新七条数据 + date1=list1[-1] + date1= date1 + ' ' + times + timeArray = time.strptime(date1, "%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timeArray) + date1 = int(timestamp * 1000) - 28800000 + value1=list2[-1] + + date2=list1[-2] + date2 = date2 + ' ' + times + timeArray = time.strptime(date2, "%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timeArray) + date2 = int(timestamp * 1000) - 28800000 + value2=list2[-2] + + date3=list1[-3] + date3 = date3 + ' ' + times + timeArray = time.strptime(date3, "%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timeArray) + date3 = int(timestamp * 1000) - 28800000 + value3=list2[-3] + + date4=list1[-4] + date4 = date4 + ' ' + times + timeArray = time.strptime(date4, "%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timeArray) + date4 = int(timestamp * 1000) - 28800000 + value4=list2[-4] + + date5=list1[-5] + date5 = date5 + ' ' + times + timeArray = time.strptime(date5, "%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timeArray) + date5 = int(timestamp * 1000) - 28800000 + value5=list2[-5] + + date6=list1[-6] + date6 = date6 + ' ' + times + timeArray = time.strptime(date6, "%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timeArray) + date6 = int(timestamp * 1000) - 28800000 + value6=list2[-6] + + date7=list1[-7] + date7 = date7 + ' ' + times + timeArray = time.strptime(date7, "%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timeArray) + date7 = int(timestamp * 1000) - 28800000 + value7=list2[-7] + + db = pymysql.connect(host="127.0.0.1",user="root",password="2GS@bPYcgiMyL14A",database="Macroeconomics",port=4423) + cursor = db.cursor() + sql = "select times from WEI order by times desc limit 1" + cursor.execute(sql) + db.commit() + old_time = cursor.fetchall() + old_time = old_time[0][0] + sql = "update WEI set WEI=%s where times='%s'" % (value2, date2) + cursor.execute(sql) + sql = "update WEI set WEI=%s where times='%s'" % (value3, date3) + cursor.execute(sql) + sql = "update WEI set WEI=%s where times='%s'" % (value4, date4) + cursor.execute(sql) + sql = "update WEI set WEI=%s where times='%s'" % (value5, date5) + cursor.execute(sql) + sql = "update WEI set WEI=%s where times='%s'" % (value6, date6) + cursor.execute(sql) + sql = "update WEI set WEI=%s where times='%s'" % (value7, date7) + cursor.execute(sql) + db.commit() + if date1 == old_time: + time.sleep(21600) + # time.sleep(timer_start_time) + else: + db = pymysql.connect(host="127.0.0.1",user="root",password="2GS@bPYcgiMyL14A",database="Macroeconomics",port=4423) + cursor = db.cursor() + sql = "insert into WEI(times,WEI)values('%s','%s')" % (date1, value1) + cursor.execute(sql) + db.commit() \ No newline at end of file