案源机会的利好消息的4个维度：招投标、购地信息、地块公示、土地转让

src/main/resources/odps-schema.yaml

@@ -1,112 +1,163 @@
 job:
-  #------<公司基本信息
+#  #------<公司基本信息
+#  - project: winhc_test
+#    flow: inc_company_spark
+#    task:
+#      - taskName: company_inc
+#        param:
+#          - _nodeId: 700003381602
+#            project: winhc_eci_dev
+#            bizdate: 20200618
+#  #------>
+##------<环保处罚
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_env_punishment
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_env_punishment
+#            DIM_COLUMS: id,name,department,publish_time,punish_number,punish_basis,law_break,reason,content,execution,source_url,source_flag,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,name,source_url,punish_number
+#            DS: 20200605
+##------>
+##------<行政处罚-信用中国
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_punishment_info_creditchina
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_punishment_info_creditchina
+#            DIM_COLUMS: id,company_name,punish_number,punish_name,punish_status,legal_name,type,type_2,result,reason,evidence,area,department,decision_date,term,status,appro_time,source,source_flag,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,company_name,source,punish_number
+#            DS: 20200605
+##------>
+##------<行政处罚
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_punishment_info
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_punishment_info
+#            DIM_COLUMS: id,base,punish_number,name,reg_number,legal_person_name,legal_name_id,legal_name_type,type,content,department_name,decision_date,publish_date,description,desc_file_path,desc_oss_path,punish_desc_mod_time,punish_desc_remark,source,tag,remark,remark_clean,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,name,source,desc_file_path,punish_number
+#            DS: 20200605
+##------>
+##------<地块公示:ODPS SQL
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_land_publicity
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_land_publicity
+#            DIM_COLUMS: id,title,district,land_num,location,use_for,area,project_name,application_name,transfer_period,final_price,remark,public_announcement_period,feedback_method,contact_organize,organize_location,postal_code,contact_number,contact_person,electronic_mail,publication_organize,publication_date,publication_start_date,publication_end_date,source_url,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,title,project_name,source_url
+#            DS: 20200605
+##------>
+#------<地块公示:Spark
   - project: winhc_test
-    flow: inc_company_spark
-    task:
-      - taskName: company_inc
-        param:
-          - _nodeId: 700003381602
-            project: winhc_eci_dev
-            bizdate: 20200618
-  #------>
-#------<环保处罚
-  - project: winhc_test
-    flow: inc_cid_sql
-    task:
-      - taskName: company_env_punishment
-        param:
-          - _nodeId: 700003375028
-            PROJECT: winhc_eci_dev
-            DIM_TABLE: company_env_punishment
-            DIM_COLUMS: id,name,department,publish_time,punish_number,punish_basis,law_break,reason,content,execution,source_url,source_flag,create_time,update_time,deleted
-            DUPLI_COLS: new_cid,name,source_url,punish_number
-            DS: 20200605
-#------>
-#------<行政处罚-信用中国
-  - project: winhc_test
-    flow: inc_cid_sql
-    task:
-      - taskName: company_punishment_info_creditchina
-        param:
-          - _nodeId: 700003375028
-            PROJECT: winhc_eci_dev
-            DIM_TABLE: company_punishment_info_creditchina
-            DIM_COLUMS: id,company_name,punish_number,punish_name,punish_status,legal_name,type,type_2,result,reason,evidence,area,department,decision_date,term,status,appro_time,source,source_flag,create_time,update_time,deleted
-            DUPLI_COLS: new_cid,company_name,source,punish_number
-            DS: 20200605
-#------>
-#------<行政处罚
-  - project: winhc_test
-    flow: inc_cid_sql
-    task:
-      - taskName: company_punishment_info
-        param:
-          - _nodeId: 700003375028
-            PROJECT: winhc_eci_dev
-            DIM_TABLE: company_punishment_info
-            DIM_COLUMS: id,base,punish_number,name,reg_number,legal_person_name,legal_name_id,legal_name_type,type,content,department_name,decision_date,publish_date,description,desc_file_path,desc_oss_path,punish_desc_mod_time,punish_desc_remark,source,tag,remark,remark_clean,create_time,update_time,deleted
-            DUPLI_COLS: new_cid,name,source,desc_file_path,punish_number
-            DS: 20200605
-#------>
-#------<地块公示
-  - project: winhc_test
-    flow: inc_cid_sql
+    flow: incr_calc_intellectual
     task:
       - taskName: company_land_publicity
         param:
-          - _nodeId: 700003375028
-            PROJECT: winhc_eci_dev
-            DIM_TABLE: company_land_publicity
-            DIM_COLUMS: id,title,district,land_num,location,use_for,area,project_name,application_name,transfer_period,final_price,remark,public_announcement_period,feedback_method,contact_organize,organize_location,postal_code,contact_number,contact_person,electronic_mail,publication_organize,publication_date,publication_start_date,publication_end_date,source_url,create_time,update_time,deleted
-            DUPLI_COLS: new_cid,title,project_name,source_url
-            DS: 20200605
+          - _nodeId: 700003375026
+            project: winhc_eci_dev
+            tableName: company_land_publicity
+            dupliCols: new_cid,title,project_name,source_url
+            flag: cid
+          - _nodeId: 700003380225
+            project: winhc_eci_dev
+            tableName: company_land_publicity
+            cidField: new_cid
+            dupliCols: new_cid,title,project_name,source_url
 #------>
-#------<招投标
+##------<招投标:ODPS SQL
+#  - project: winhc_test
+#    flow: inc_cids_sql
+#    task:
+#      - taskName: company_bid
+#        param:
+#          - _nodeId: 700003375027
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_bid
+#            DIM_COLUMS: id,title,link,intro,abs,publish_time,purchaser,proxy,province,base,type,items,create_time,update_time,deleted
+#            DIM_LIST_COLUMS: id,title,publish_time,province,base,type,items,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,title,publish_time
+#            DS: 20200605
+##------>
+#------<招投标:Spark
   - project: winhc_test
-    flow: inc_cids_sql
+    flow: incr_calc_intellectual
     task:
       - taskName: company_bid
         param:
-          - _nodeId: 700003375027
-            PROJECT: winhc_eci_dev
-            DIM_TABLE: company_bid
-            DIM_COLUMS: id,title,link,intro,abs,publish_time,purchaser,proxy,province,base,type,items,create_time,update_time,deleted
-            DIM_LIST_COLUMS: id,title,publish_time,province,base,type,items,create_time,update_time,deleted
-            DUPLI_COLS: new_cid,id
-            DS: 20200605
+          - _nodeId: 700003375026
+            project: winhc_eci_dev
+            tableName: company_bid
+            dupliCols: new_cid,title,publish_time
+            flag: cids
+          - _nodeId: 700003380225
+            project: winhc_eci_dev
+            tableName: company_bid_list
+            cidField: new_cid
+            dupliCols: new_cid,title,publish_time
 #------>
-#------<购地信息
+##------<购地信息:ODPS SQL
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_land_announcement
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_land_announcement
+#            DIM_COLUMS: id,e_number,application_name,district,project_name,project_loc,area,source,source_view,use_type,supply_method,use_period,category,land_level,tran_price,volume_rate,volume_rate_ceiling,commit_time,agreement_start_time,actual_start_time,scheduled_completion,actual_completion_time,authority,contract_date,instalment_payment,source_url,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,source,e_number,application_name
+#            DS: 20200605
+##------>
+#------<购地信息:Spark
   - project: winhc_test
-    flow: inc_cid_sql
+    flow: incr_calc_intellectual
     task:
       - taskName: company_land_announcement
         param:
-          - _nodeId: 700003375028
-            PROJECT: winhc_eci_dev
-            DIM_TABLE: company_land_announcement
-            DIM_COLUMS: id,e_number,application_name,district,project_name,project_loc,area,source,source_view,use_type,supply_method,use_period,category,land_level,tran_price,volume_rate,volume_rate_ceiling,commit_time,agreement_start_time,actual_start_time,scheduled_completion,actual_completion_time,authority,contract_date,instalment_payment,source_url,create_time,update_time,deleted
-            DUPLI_COLS: new_cid,source,e_number,application_name
-            DS: 20200605
+          - _nodeId: 700003375026
+            project: winhc_eci_dev
+            tableName: company_land_announcement
+            dupliCols: new_cid,source,e_number,application_name
+            flag: cid
+          - _nodeId: 700003380225
+            project: winhc_eci_dev
+            tableName: company_land_announcement
+            cidField: new_cid
+            dupliCols: new_cid,source,e_number,application_name
 #------>
 #------<土地抵押
-  - project: winhc_test
-    flow: inc_company_land_mortgage_sql
-    task:
-      - taskName: company_land_mortgage
-        param:
-          - _nodeId: 700003375909
-            PROJECT: winhc_eci_dev
-            DIM_TABLE: company_land_mortgage
-            DIM_COLUMS: type,id,land_mark,land_num,land_aministrative_area,land_loc,land_area,other_item_num,use_right_num,mortgagor,mortgagee,nature,use_for,use_type,area,evaluate_amount,mortgage_amount,source_url,start_date,end_date,create_time,update_time,deleted
-            DUPLI_COLS: new_cid,type,source_url,land_mark,land_num
-            DS: 20200605
-          - _nodeId: 700003375910
-            PROJECT: winhc_test_dev
-            DIM_TABLE: company_land_mortgage
-            DIM_COLUMS: type,id,land_mark,land_num,land_aministrative_area,land_loc,land_area,other_item_num,use_right_num,mortgagor,mortgagee,nature,use_for,use_type,area,evaluate_amount,mortgage_amount,source_url,start_date,end_date,create_time,update_time,deleted
-            DUPLI_COLS: new_cid,type,source_url,land_mark,land_num
-            DS: 20200605
-#------>
+#  - project: winhc_test
+#    flow: inc_company_land_mortgage_sql
+#    task:
+#      - taskName: company_land_mortgage
+#        param:
+#          - _nodeId: 700003375909
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_land_mortgage
+#            DIM_COLUMS: type,id,land_mark,land_num,land_aministrative_area,land_loc,land_area,other_item_num,use_right_num,mortgagor,mortgagee,nature,use_for,use_type,area,evaluate_amount,mortgage_amount,source_url,start_date,end_date,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,type,source_url,land_mark,land_num
+#            DS: 20200605
+#          - _nodeId: 700003375910
+#            PROJECT: winhc_test_dev
+#            DIM_TABLE: company_land_mortgage
+#            DIM_COLUMS: type,id,land_mark,land_num,land_aministrative_area,land_loc,land_area,other_item_num,use_right_num,mortgagor,mortgagee,nature,use_for,use_type,area,evaluate_amount,mortgage_amount,source_url,start_date,end_date,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,type,source_url,land_mark,land_num
+#            DS: 20200605
+##------>
 #------<土地转让
   - project: winhc_test
     flow: inc_company_land_transfer_sql