{"description":"实验创建于2022/11/10","graph":{"edges":[{"to_node_id":"-183:input_data","from_node_id":"-314:data_1"}],"nodes":[{"node_id":"-183","module_id":"BigQuantSpace.datahub_update_datasource.datahub_update_datasource-v5","parameters":[{"name":"alias","value":"hxie_crawl_demos","type":"Literal","bound_global_parameter":null},{"name":"primary_key","value":"date\ninstrument","type":"Literal","bound_global_parameter":null},{"name":"date_field","value":"date","type":"Literal","bound_global_parameter":null},{"name":"partition_date","value":"年","type":"Literal","bound_global_parameter":null},{"name":"rewrite","value":"False","type":"Literal","bound_global_parameter":null},{"name":"public","value":"True","type":"Literal","bound_global_parameter":null},{"name":"friendly_name","value":"东方财富爬虫","type":"Literal","bound_global_parameter":null},{"name":"desc","value":"东方财富爬虫","type":"Literal","bound_global_parameter":null},{"name":"only_desc_fields","value":"False","type":"Literal","bound_global_parameter":null},{"name":"fields","value":"\n# #号开始的表示注释，注释需单独一行\n# 对您数据中的每个字段进行描述，系统会根据您定义的数据类型自动将对应列的数据转换\n# 如果是已有数据表，默认会获取已有表的所有数据字段信息\n# eg: {'date': {'desc': '描述', 'type': '数据类型'}, ...}\n# {'instrument': {'desc': '证券代码', 'type': 'str'}, ... }\n{}\n","type":"Literal","bound_global_parameter":null},{"name":"show_doc","value":"True","type":"Literal","bound_global_parameter":null}],"input_ports":[{"name":"input_data","node_id":"-183"}],"output_ports":[{"name":"data","node_id":"-183"}],"cacheable":false,"seq_num":1,"comment":"","comment_collapsed":true},{"node_id":"-34","module_id":"BigQuantSpace.datahub_usertask2.datahub_usertask2-v2","parameters":[{"name":"task_name","value":"hxie_crawl__task","type":"Literal","bound_global_parameter":null},{"name":"description","value":"东方财富爬虫","type":"Literal","bound_global_parameter":null},{"name":"run_time","value":"17:00","type":"Literal","bound_global_parameter":null},{"name":"day_of_month","value":"每天","type":"Literal","bound_global_parameter":null},{"name":"run_weekly","value":"True","type":"Literal","bound_global_parameter":null},{"name":"update_mode","value":"False","type":"Literal","bound_global_parameter":null},{"name":"run_now","value":"True","type":"Literal","bound_global_parameter":null}],"input_ports":[],"output_ports":[{"name":"task","node_id":"-34"}],"cacheable":false,"seq_num":2,"comment":"","comment_collapsed":true},{"node_id":"-314","module_id":"BigQuantSpace.cached.cached-v3","parameters":[{"name":"run","value":"# Python 代码入口函数，input_1/2/3 对应三个输入端，data_1/2/3 对应三个输出端\ndef bigquant_run(input_1, input_2, input_3):\n import requests\n import re\n\n cookies = {\n 'qgqp_b_id': '02d480cce140d4a420a0df6b307a945c',\n 'cowCookie': 'true',\n 'em_hq_fls': 'js',\n 'intellpositionL': '1168.61px',\n 'HAList': 'a-sz-300059-%u4E1C%u65B9%u8D22%u5BCC%2Ca-sz-000001-%u5E73%u5B89%u94F6%u884C',\n 'st_si': '07441051579204',\n 'st_asi': 'delete',\n 'st_pvi': '34234318767565',\n 'st_sp': '2021-09-28%2010%3A43%3A13',\n 'st_inirUrl': 'http%3A%2F%2Fdata.eastmoney.com%2F',\n 'st_sn': '31',\n 'st_psi': '20211020210419860-113300300813-5631892871',\n 'intellpositionT': '1007.88px',\n }\n\n headers = {\n 'Connection': 'keep-alive',\n 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36 Edg/94.0.992.50',\n 'DNT': '1',\n 'Accept': '*/*',\n 'Referer': 'http://quote.eastmoney.com/',\n 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',\n }\n\n params = (\n ('cb', 'jQuery112404825022376475756_1634735261901'),\n ('pn', '1'),\n ('pz', '20'),\n ('po', '1'),\n ('np', '1'),\n ('ut', 'bd1d9ddb04089700cf9c27f6f7426281'),\n ('fltt', '2'),\n ('invt', '2'),\n ('fid', 'f3'),\n ('fs', 'm:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23'),\n ('fields', 'f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152'),\n ('_', '1634735261902'),\n )\n\n response = requests.get('http://54.push2.eastmoney.com/api/qt/clist/get', headers=headers, params=params, cookies=cookies, verify=False)\n\n for page in range(1,50):\n params = (\n ('cb', 'jQuery1124031167968836399784_1615878909521'),\n ('pn', str(page)),\n ('pz', '20'),\n ('po', '1'),\n ('np', '1'),\n ('ut', 'bd1d9ddb04089700cf9c27f6f7426281'),\n ('fltt', '2'),\n ('invt', '2'),\n ('fid', 'f3'),\n ('fs', 'm:0 t:6,m:0 t:13,m:0 t:80,m:1 t:2,m:1 t:23'),\n ('fields', 'f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152'),\n )\n\n daimas = re.findall('\"f12\":(.*?),',response.text)\n names = re.findall('\"f14\":\"(.*?)\"',response.text)\n zuixinjias = re.findall('\"f2\":(.*?),',response.text)\n zhangdiefus = re.findall('\"f3\":(.*?),',response.text)\n zhangdiees = re.findall('\"f4\":(.*?),',response.text) \n chengjiaoliangs = re.findall('\"f5\":(.*?),',response.text) \n chengjiaoes = re.findall('\"f6\":(.*?),',response.text)\n zhenfus = re.findall('\"f7\":(.*?),',response.text) \n zuigaos = re.findall('\"f15\":(.*?),',response.text) \n zuidis = re.findall('\"f16\":(.*?),',response.text) \n jinkais = re.findall('\"f17\":(.*?),',response.text) \n zuoshous = re.findall('\"f18\":(.*?),',response.text) \n liangbis = re.findall('\"f10\":(.*?),',response.text) \n huanshoulvs = re.findall('\"f8\":(.*?),',response.text) \n shiyinglvs = re.findall('\"f9\":(.*?),',response.text)\n\n ins = list()\n for i in daimas:\n i = i[1:-1]\n if i.startswith('6'):\n i = i+'.SHA'\n elif i.startswith('3') or i.startswith('0'):\n i = i+'.SZA'\n elif i.startswith('8'):\n i = i+'.BJA'\n ins.append(i)\n\n df = pd.DataFrame({\n 'instrument': ins,\n 'names': names,\n 'lastprice': zuixinjias,\n 'zhangdiefu': zhangdiefus,\n 'zhangdiees': zhangdiees,\n 'chengjiaoliangs': chengjiaoliangs,\n 'chengjiaoes': chengjiaoes,\n 'zhenfus': zhenfus,\n 'zuigaos': zuigaos,\n 'zuidis': zuidis,\n 'jinkais': jinkais,\n 'zuoshous': zuoshous,\n 'liangbis': liangbis,\n 'huanshoulvs': huanshoulvs,\n 'shiyinglvs': shiyinglvs,\n })\n from datetime import datetime\n\n df['date'] = datetime.now().strftime('%Y-%m-%d')\n\n data_1 = DataSource.write_df(df)\n return Outputs(data_1=data_1, data_2=None, data_3=None)\n","type":"Literal","bound_global_parameter":null},{"name":"post_run","value":"# 后处理函数，可选。输入是主函数的输出，可以在这里对数据做处理，或者返回更友好的outputs数据格式。此函数输出不会被缓存。\ndef bigquant_run(outputs):\n return outputs\n","type":"Literal","bound_global_parameter":null},{"name":"input_ports","value":"","type":"Literal","bound_global_parameter":null},{"name":"params","value":"{}","type":"Literal","bound_global_parameter":null},{"name":"output_ports","value":"","type":"Literal","bound_global_parameter":null}],"input_ports":[{"name":"input_1","node_id":"-314"},{"name":"input_2","node_id":"-314"},{"name":"input_3","node_id":"-314"}],"output_ports":[{"name":"data_1","node_id":"-314"},{"name":"data_2","node_id":"-314"},{"name":"data_3","node_id":"-314"}],"cacheable":true,"seq_num":3,"comment":"","comment_collapsed":true}],"node_layout":"<node_postions><node_position Node='-183' Position='7,0,200,200'/><node_position Node='-34' Position='0,98,200,200'/><node_position Node='-314' Position='9.879158020019531,-114.63839247822762,200,200'/></node_postions>"},"nodes_readonly":false,"studio_version":"v2"}

# 本代码由可视化策略环境自动生成 2022年11月10日 15:57
# 本代码单元只能在可视化模式下编辑。您也可以拷贝代码，粘贴到新建的代码单元或者策略，然后修改。


# Python 代码入口函数，input_1/2/3 对应三个输入端，data_1/2/3 对应三个输出端
def m3_run_bigquant_run(input_1, input_2, input_3):
    import requests
    import re

    cookies = {
        'qgqp_b_id': '02d480cce140d4a420a0df6b307a945c',
        'cowCookie': 'true',
        'em_hq_fls': 'js',
        'intellpositionL': '1168.61px',
        'HAList': 'a-sz-300059-%u4E1C%u65B9%u8D22%u5BCC%2Ca-sz-000001-%u5E73%u5B89%u94F6%u884C',
        'st_si': '07441051579204',
        'st_asi': 'delete',
        'st_pvi': '34234318767565',
        'st_sp': '2021-09-28%2010%3A43%3A13',
        'st_inirUrl': 'http%3A%2F%2Fdata.eastmoney.com%2F',
        'st_sn': '31',
        'st_psi': '20211020210419860-113300300813-5631892871',
        'intellpositionT': '1007.88px',
    }

    headers = {
        'Connection': 'keep-alive',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36 Edg/94.0.992.50',
        'DNT': '1',
        'Accept': '*/*',
        'Referer': 'http://quote.eastmoney.com/',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    }

    params = (
        ('cb', 'jQuery112404825022376475756_1634735261901'),
        ('pn', '1'),
        ('pz', '20'),
        ('po', '1'),
        ('np', '1'),
        ('ut', 'bd1d9ddb04089700cf9c27f6f7426281'),
        ('fltt', '2'),
        ('invt', '2'),
        ('fid', 'f3'),
        ('fs', 'm:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23'),
        ('fields', 'f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152'),
        ('_', '1634735261902'),
    )

    response = requests.get('http://54.push2.eastmoney.com/api/qt/clist/get', headers=headers, params=params, cookies=cookies, verify=False)

    for page in range(1,50):
            params = (
                ('cb', 'jQuery1124031167968836399784_1615878909521'),
                ('pn', str(page)),
                ('pz', '20'),
                ('po', '1'),
                ('np', '1'),
                ('ut', 'bd1d9ddb04089700cf9c27f6f7426281'),
                ('fltt', '2'),
                ('invt', '2'),
                ('fid', 'f3'),
                ('fs', 'm:0 t:6,m:0 t:13,m:0 t:80,m:1 t:2,m:1 t:23'),
                ('fields', 'f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152'),
            )

    daimas = re.findall('"f12":(.*?),',response.text)
    names = re.findall('"f14":"(.*?)"',response.text)
    zuixinjias = re.findall('"f2":(.*?),',response.text)
    zhangdiefus = re.findall('"f3":(.*?),',response.text)
    zhangdiees = re.findall('"f4":(.*?),',response.text)      
    chengjiaoliangs = re.findall('"f5":(.*?),',response.text)       
    chengjiaoes = re.findall('"f6":(.*?),',response.text)
    zhenfus = re.findall('"f7":(.*?),',response.text)       
    zuigaos = re.findall('"f15":(.*?),',response.text)        
    zuidis = re.findall('"f16":(.*?),',response.text)        
    jinkais = re.findall('"f17":(.*?),',response.text)        
    zuoshous = re.findall('"f18":(.*?),',response.text)       
    liangbis = re.findall('"f10":(.*?),',response.text)        
    huanshoulvs = re.findall('"f8":(.*?),',response.text)       
    shiyinglvs = re.findall('"f9":(.*?),',response.text)

    ins = list()
    for i in daimas:
        i = i[1:-1]
        if i.startswith('6'):
            i = i+'.SHA'
        elif i.startswith('3') or i.startswith('0'):
            i = i+'.SZA'
        elif i.startswith('8'):
            i = i+'.BJA'
        ins.append(i)

    df = pd.DataFrame({
        'instrument': ins,
        'names': names,
        'lastprice': zuixinjias,
        'zhangdiefu': zhangdiefus,
        'zhangdiees': zhangdiees,
        'chengjiaoliangs': chengjiaoliangs,
        'chengjiaoes': chengjiaoes,
        'zhenfus': zhenfus,
        'zuigaos': zuigaos,
        'zuidis': zuidis,
        'jinkais': jinkais,
        'zuoshous': zuoshous,
        'liangbis': liangbis,
        'huanshoulvs': huanshoulvs,
        'shiyinglvs': shiyinglvs,
    })
    from datetime import datetime

    df['date'] = datetime.now().strftime('%Y-%m-%d')

    data_1 = DataSource.write_df(df)
    return Outputs(data_1=data_1, data_2=None, data_3=None)

# 后处理函数，可选。输入是主函数的输出，可以在这里对数据做处理，或者返回更友好的outputs数据格式。此函数输出不会被缓存。
def m3_post_run_bigquant_run(outputs):
    return outputs


m2 = M.datahub_usertask2.v2(
    task_name='hxie_crawl__task',
    description='东方财富爬虫',
    run_time='17:00',
    day_of_month='每天',
    run_weekly=True,
    update_mode=False,
    run_now=True
)

m3 = M.cached.v3(
    run=m3_run_bigquant_run,
    post_run=m3_post_run_bigquant_run,
    input_ports='',
    params='{}',
    output_ports=''
)

m1 = M.datahub_update_datasource.v5(
    input_data=m3.data_1,
    alias='hxie_crawl_demos',
    primary_key="""date
instrument""",
    date_field='date',
    partition_date='年',
    rewrite=False,
    public=True,
    friendly_name='东方财富爬虫',
    desc='东方财富爬虫',
    only_desc_fields=False,
    fields=
# #号开始的表示注释，注释需单独一行
# 对您数据中的每个字段进行描述，系统会根据您定义的数据类型自动将对应列的数据转换
# 如果是已有数据表，默认会获取已有表的所有数据字段信息
# eg: {'date': {'desc': '描述', 'type': '数据类型'}, ...}
# {'instrument': {'desc': '证券代码', 'type': 'str'}, ... }
{}
,
    show_doc=True
)

[2022-11-10 15:57:04.626173] INFO: 数据平台任务(创建/更新): 任务名称: hxie_crawl__task 定时设置: 0 17 * * * 任务描述: 东方财富爬虫

[2022-11-10 15:57:04.722509] INFO: 数据平台任务(创建/更新): {'code': 0, 'reason': 'OK', 'message': 'OK', 'data': {'code': '200', 'msg': 'success create task'}}

[2022-11-10 15:57:04.724623] INFO: moduleinvoker: datahub_usertask2.v2 运行完成[0.098662s].

[2022-11-10 15:57:04.740365] INFO: moduleinvoker: cached.v3 开始运行..

[2022-11-10 15:57:04.754616] INFO: moduleinvoker: 命中缓存

[2022-11-10 15:57:04.756685] INFO: moduleinvoker: cached.v3 运行完成[0.016325s].

[2022-11-10 15:57:05.054112] INFO: 更新入库: 更新数据 1/1 

[2022-11-10 15:57:20.411446] INFO: 更新入库: 更新完成 1/1 ... 

[2022-11-10 15:57:20.413939] INFO: 更新入库: 数据更新完成, 可以使用  DataSource('hxie_crawl_demos_U').read()  来查看数据!

[2022-11-10 15:57:20.416385] INFO: moduleinvoker: datahub_update_datasource.v5 运行完成[15.649259s].