feat(etl): dataX

org测试流程,dataX项目导出
4 weeks ago · 648f690252
parent 5b2cd2c5d9
commit 648f690252
2 changed files with 134 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,33 @@
+HELP.md
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
--- a/etl_org/datax/etl_org_subject_article_data_hdfs2mysql.json
+++ b/etl_org/datax/etl_org_subject_article_data_hdfs2mysql.json
@ -0,0 +1,101 @@
+{
+    "job": {
+        "setting": {
+            "speed": {
+                "channel": 1
+            }
+        },
+        "content": [
+            {
+                "reader": {
+                    "name": "hdfsreader",
+                    "csvReaderConfig": {
+                        "safetySwitch": false,
+                        "skipEmptyRecords": false,
+                        "useTextQualifier": false
+                    },
+                    "parameter": {
+                        "path": "/user/hive/warehouse/science_etl_dws.db/etl_org_subject_article_data/*",
+                        "defaultFS": "hdfs://hadoop01:8020",
+                        "column": [
+                            {
+                                "index": 0,
+                                "type": "String"
+                            },
+                            {
+                                "index": 1,
+                                "type": "String"
+                            },
+                            {
+                                "index": 2,
+                                "type": "String"
+                            },
+                            {
+                                "index": 3,
+                                "type": "String"
+                            },
+                            {
+                                "index": 4,
+                                "type": "String"
+                            },
+                            {
+                                "index": 5,
+                                "type": "String"
+                            },
+                            {
+                                "index": 6,
+                                "type": "String"
+                            },
+                            {
+                                "index": 7,
+                                "type": "String"
+                            },
+                            {
+                                "index": 8,
+                                "type": "String"
+                            },
+                            {
+                                "index": 9,
+                                "type": "String"
+                            }
+                        ],
+                        "fileType": "orc",
+                        "encoding": "UTF-8",
+                        "fieldDelimiter": "\t"
+                    }
+                },
+                "writer": {
+                    "name": "mysqlwriter",
+                    "parameter": {
+                        "writeMode": "insert",
+                        "username": "root",
+                        "password": "123456",
+                        "column": [
+                            "source_type",
+                            "school_id",
+                            "subject_id",
+                            "org_id",
+                            "article_count",
+                            "first_count",
+                            "rep_count",
+                            "cited_count",
+                            "first_cited_count",
+                            "rep_cited_count"
+                        ],
+                        "preSql": [
+                            "delete from etl_org_subject_article_data"
+                        ],
+                        "connection": [
+                            {
+                                "jdbcUrl": "jdbc:mysql://192.168.1.31:3305/datax?characterEncoding=UTF-8&connectionCollation=utf8mb4_general_ci&useUnicode=true&useSSL=false&allowMultiQueries=true&serverTimezone=GMT%2b8",
+                                "table": [
+                                    "etl_org_subject_article_data"
+                                ]
+                            }
+                        ]
+                    }
+                }
+            }
+        ]
+    }
+}