init

.gitignore

@@ -0,0 +1,33 @@
+HELP.md
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/

.mvn/wrapper/MavenWrapperDownloader.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2007-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.net.*;
+import java.io.*;
+import java.nio.channels.*;
+import java.util.Properties;
+
+public class MavenWrapperDownloader {
+
+    private static final String WRAPPER_VERSION = "0.5.6";
+    /**
+     * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
+     */
+    private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+            + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
+
+    /**
+     * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
+     * use instead of the default one.
+     */
+    private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
+            ".mvn/wrapper/maven-wrapper.properties";
+
+    /**
+     * Path where the maven-wrapper.jar will be saved to.
+     */
+    private static final String MAVEN_WRAPPER_JAR_PATH =
+            ".mvn/wrapper/maven-wrapper.jar";
+
+    /**
+     * Name of the property which should be used to override the default download url for the wrapper.
+     */
+    private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
+
+    public static void main(String args[]) {
+        System.out.println("- Downloader started");
+        File baseDirectory = new File(args[0]);
+        System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
+
+        // If the maven-wrapper.properties exists, read it and check if it contains a custom
+        // wrapperUrl parameter.
+        File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
+        String url = DEFAULT_DOWNLOAD_URL;
+        if (mavenWrapperPropertyFile.exists()) {
+            FileInputStream mavenWrapperPropertyFileInputStream = null;
+            try {
+                mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
+                Properties mavenWrapperProperties = new Properties();
+                mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
+                url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
+            } catch (IOException e) {
+                System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
+            } finally {
+                try {
+                    if (mavenWrapperPropertyFileInputStream != null) {
+                        mavenWrapperPropertyFileInputStream.close();
+                    }
+                } catch (IOException e) {
+                    // Ignore ...
+                }
+            }
+        }
+        System.out.println("- Downloading from: " + url);
+
+        File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
+        if (!outputFile.getParentFile().exists()) {
+            if (!outputFile.getParentFile().mkdirs()) {
+                System.out.println(
+                        "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
+            }
+        }
+        System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
+        try {
+            downloadFileFromURL(url, outputFile);
+            System.out.println("Done");
+            System.exit(0);
+        } catch (Throwable e) {
+            System.out.println("- Error downloading");
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private static void downloadFileFromURL(String urlString, File destination) throws Exception {
+        if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
+            String username = System.getenv("MVNW_USERNAME");
+            char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
+            Authenticator.setDefault(new Authenticator() {
+                @Override
+                protected PasswordAuthentication getPasswordAuthentication() {
+                    return new PasswordAuthentication(username, password);
+                }
+            });
+        }
+        URL website = new URL(urlString);
+        ReadableByteChannel rbc;
+        rbc = Channels.newChannel(website.openStream());
+        FileOutputStream fos = new FileOutputStream(destination);
+        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+        fos.close();
+        rbc.close();
+    }
+
+}

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,2 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.1/apache-maven-3.8.1-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

mvnw

@@ -0,0 +1,322 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Maven Start Up Batch script
+#
+# Required ENV vars:
+# ------------------
+#   JAVA_HOME - location of a JDK home dir
+#
+# Optional ENV vars
+# -----------------
+#   M2_HOME - location of maven2's installed home dir
+#   MAVEN_OPTS - parameters passed to the Java VM when running Maven
+#     e.g. to debug Maven itself, use
+#       set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+#   MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# ----------------------------------------------------------------------------
+
+if [ -z "$MAVEN_SKIP_RC" ]; then
+
+  if [ -f /etc/mavenrc ]; then
+    . /etc/mavenrc
+  fi
+
+  if [ -f "$HOME/.mavenrc" ]; then
+    . "$HOME/.mavenrc"
+  fi
+
+fi
+
+# OS specific support.  $var _must_ be set to either true or false.
+cygwin=false
+darwin=false
+mingw=false
+case "$(uname)" in
+CYGWIN*) cygwin=true ;;
+MINGW*) mingw=true ;;
+Darwin*)
+  darwin=true
+  # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
+  # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
+  if [ -z "$JAVA_HOME" ]; then
+    if [ -x "/usr/libexec/java_home" ]; then
+      export JAVA_HOME="$(/usr/libexec/java_home)"
+    else
+      export JAVA_HOME="/Library/Java/Home"
+    fi
+  fi
+  ;;
+esac
+
+if [ -z "$JAVA_HOME" ]; then
+  if [ -r /etc/gentoo-release ]; then
+    JAVA_HOME=$(java-config --jre-home)
+  fi
+fi
+
+if [ -z "$M2_HOME" ]; then
+  ## resolve links - $0 may be a link to maven's home
+  PRG="$0"
+
+  # need this for relative symlinks
+  while [ -h "$PRG" ]; do
+    ls=$(ls -ld "$PRG")
+    link=$(expr "$ls" : '.*-> \(.*\)$')
+    if expr "$link" : '/.*' >/dev/null; then
+      PRG="$link"
+    else
+      PRG="$(dirname "$PRG")/$link"
+    fi
+  done
+
+  saveddir=$(pwd)
+
+  M2_HOME=$(dirname "$PRG")/..
+
+  # make it fully qualified
+  M2_HOME=$(cd "$M2_HOME" && pwd)
+
+  cd "$saveddir"
+  # echo Using m2 at $M2_HOME
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin; then
+  [ -n "$M2_HOME" ] &&
+    M2_HOME=$(cygpath --unix "$M2_HOME")
+  [ -n "$JAVA_HOME" ] &&
+    JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
+  [ -n "$CLASSPATH" ] &&
+    CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
+fi
+
+# For Mingw, ensure paths are in UNIX format before anything is touched
+if $mingw; then
+  [ -n "$M2_HOME" ] &&
+    M2_HOME="$( (
+      cd "$M2_HOME"
+      pwd
+    ))"
+  [ -n "$JAVA_HOME" ] &&
+    JAVA_HOME="$( (
+      cd "$JAVA_HOME"
+      pwd
+    ))"
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+  javaExecutable="$(which javac)"
+  if [ -n "$javaExecutable" ] && ! [ "$(expr \"$javaExecutable\" : '\([^ ]*\)')" = "no" ]; then
+    # readlink(1) is not available as standard on Solaris 10.
+    readLink=$(which readlink)
+    if [ ! $(expr "$readLink" : '\([^ ]*\)') = "no" ]; then
+      if $darwin; then
+        javaHome="$(dirname \"$javaExecutable\")"
+        javaExecutable="$(cd \"$javaHome\" && pwd -P)/javac"
+      else
+        javaExecutable="$(readlink -f \"$javaExecutable\")"
+      fi
+      javaHome="$(dirname \"$javaExecutable\")"
+      javaHome=$(expr "$javaHome" : '\(.*\)/bin')
+      JAVA_HOME="$javaHome"
+      export JAVA_HOME
+    fi
+  fi
+fi
+
+if [ -z "$JAVACMD" ]; then
+  if [ -n "$JAVA_HOME" ]; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ]; then
+      # IBM's JDK on AIX uses strange locations for the executables
+      JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+      JAVACMD="$JAVA_HOME/bin/java"
+    fi
+  else
+    JAVACMD="$(which java)"
+  fi
+fi
+
+if [ ! -x "$JAVACMD" ]; then
+  echo "Error: JAVA_HOME is not defined correctly." >&2
+  echo "  We cannot execute $JAVACMD" >&2
+  exit 1
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+  echo "Warning: JAVA_HOME environment variable is not set."
+fi
+
+CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
+
+# traverses directory structure from process work directory to filesystem root
+# first directory with .mvn subdirectory is considered project base directory
+find_maven_basedir() {
+
+  if [ -z "$1" ]; then
+    echo "Path not specified to find_maven_basedir"
+    return 1
+  fi
+
+  basedir="$1"
+  wdir="$1"
+  while [ "$wdir" != '/' ]; do
+    if [ -d "$wdir"/.mvn ]; then
+      basedir=$wdir
+      break
+    fi
+    # workaround for JBEAP-8937 (on Solaris 10/Sparc)
+    if [ -d "${wdir}" ]; then
+      wdir=$(
+        cd "$wdir/.."
+        pwd
+      )
+    fi
+    # end of workaround
+  done
+  echo "${basedir}"
+}
+
+# concatenates all lines of a file
+concat_lines() {
+  if [ -f "$1" ]; then
+    echo "$(tr -s '\n' ' ' <"$1")"
+  fi
+}
+
+BASE_DIR=$(find_maven_basedir "$(pwd)")
+if [ -z "$BASE_DIR" ]; then
+  exit 1
+fi
+
+##########################################################################################
+# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+# This allows using the maven wrapper in projects that prohibit checking in binary data.
+##########################################################################################
+if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
+  if [ "$MVNW_VERBOSE" = true ]; then
+    echo "Found .mvn/wrapper/maven-wrapper.jar"
+  fi
+else
+  if [ "$MVNW_VERBOSE" = true ]; then
+    echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
+  fi
+  if [ -n "$MVNW_REPOURL" ]; then
+    jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
+  else
+    jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
+  fi
+  while IFS="=" read key value; do
+    case "$key" in wrapperUrl)
+      jarUrl="$value"
+      break
+      ;;
+    esac
+  done <"$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
+  if [ "$MVNW_VERBOSE" = true ]; then
+    echo "Downloading from: $jarUrl"
+  fi
+  wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
+  if $cygwin; then
+    wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath")
+  fi
+
+  if command -v wget >/dev/null; then
+    if [ "$MVNW_VERBOSE" = true ]; then
+      echo "Found wget ... using wget"
+    fi
+    if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+      wget "$jarUrl" -O "$wrapperJarPath"
+    else
+      wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath"
+    fi
+  elif command -v curl >/dev/null; then
+    if [ "$MVNW_VERBOSE" = true ]; then
+      echo "Found curl ... using curl"
+    fi
+    if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+      curl -o "$wrapperJarPath" "$jarUrl" -f
+    else
+      curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
+    fi
+
+  else
+    if [ "$MVNW_VERBOSE" = true ]; then
+      echo "Falling back to using Java to download"
+    fi
+    javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
+    # For Cygwin, switch paths to Windows format before running javac
+    if $cygwin; then
+      javaClass=$(cygpath --path --windows "$javaClass")
+    fi
+    if [ -e "$javaClass" ]; then
+      if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+        if [ "$MVNW_VERBOSE" = true ]; then
+          echo " - Compiling MavenWrapperDownloader.java ..."
+        fi
+        # Compiling the Java class
+        ("$JAVA_HOME/bin/javac" "$javaClass")
+      fi
+      if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+        # Running the downloader
+        if [ "$MVNW_VERBOSE" = true ]; then
+          echo " - Running MavenWrapperDownloader.java ..."
+        fi
+        ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
+      fi
+    fi
+  fi
+fi
+##########################################################################################
+# End of extension
+##########################################################################################
+
+export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
+if [ "$MVNW_VERBOSE" = true ]; then
+  echo $MAVEN_PROJECTBASEDIR
+fi
+MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+  [ -n "$M2_HOME" ] &&
+    M2_HOME=$(cygpath --path --windows "$M2_HOME")
+  [ -n "$JAVA_HOME" ] &&
+    JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME")
+  [ -n "$CLASSPATH" ] &&
+    CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
+  [ -n "$MAVEN_PROJECTBASEDIR" ] &&
+    MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR")
+fi
+
+# Provide a "standardized" way to retrieve the CLI args that will
+# work with both Windows and non-Windows executions.
+MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
+export MAVEN_CMD_LINE_ARGS
+
+WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+exec "$JAVACMD" \
+  $MAVEN_OPTS \
+  -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
+  "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
+  ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"

mvnw.cmd

@@ -0,0 +1,182 @@
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM    https://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Maven Start Up Batch script
+@REM
+@REM Required ENV vars:
+@REM JAVA_HOME - location of a JDK home dir
+@REM
+@REM Optional ENV vars
+@REM M2_HOME - location of maven2's installed home dir
+@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
+@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
+@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
+@REM     e.g. to debug Maven itself, use
+@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+@REM ----------------------------------------------------------------------------
+
+@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
+@echo off
+@REM set title of command window
+title %0
+@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
+@if "%MAVEN_BATCH_ECHO%" == "on"  echo %MAVEN_BATCH_ECHO%
+
+@REM set %HOME% to equivalent of $HOME
+if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
+
+@REM Execute a user defined script before this one
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
+@REM check for pre script, once with legacy .bat ending and once with .cmd ending
+if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
+if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
+:skipRcPre
+
+@setlocal
+
+set ERROR_CODE=0
+
+@REM To isolate internal variables from possible post scripts, we use another setlocal
+@setlocal
+
+@REM ==== START VALIDATION ====
+if not "%JAVA_HOME%" == "" goto OkJHome
+
+echo.
+echo Error: JAVA_HOME not found in your environment. >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+:OkJHome
+if exist "%JAVA_HOME%\bin\java.exe" goto init
+
+echo.
+echo Error: JAVA_HOME is set to an invalid directory. >&2
+echo JAVA_HOME = "%JAVA_HOME%" >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+@REM ==== END VALIDATION ====
+
+:init
+
+@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
+@REM Fallback to current working directory if not found.
+
+set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
+IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
+
+set EXEC_DIR=%CD%
+set WDIR=%EXEC_DIR%
+:findBaseDir
+IF EXIST "%WDIR%"\.mvn goto baseDirFound
+cd ..
+IF "%WDIR%"=="%CD%" goto baseDirNotFound
+set WDIR=%CD%
+goto findBaseDir
+
+:baseDirFound
+set MAVEN_PROJECTBASEDIR=%WDIR%
+cd "%EXEC_DIR%"
+goto endDetectBaseDir
+
+:baseDirNotFound
+set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
+cd "%EXEC_DIR%"
+
+:endDetectBaseDir
+
+IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
+
+@setlocal EnableExtensions EnableDelayedExpansion
+for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
+@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
+
+:endReadAdditionalConfig
+
+SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
+set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
+set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
+
+FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
+    IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
+)
+
+@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
+if exist %WRAPPER_JAR% (
+    if "%MVNW_VERBOSE%" == "true" (
+        echo Found %WRAPPER_JAR%
+    )
+) else (
+    if not "%MVNW_REPOURL%" == "" (
+        SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
+    )
+    if "%MVNW_VERBOSE%" == "true" (
+        echo Couldn't find %WRAPPER_JAR%, downloading it ...
+        echo Downloading from: %DOWNLOAD_URL%
+    )
+
+    powershell -Command "&{"^
+		"$webclient = new-object System.Net.WebClient;"^
+		"if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
+		"$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
+		"}"^
+		"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^
+		"}"
+    if "%MVNW_VERBOSE%" == "true" (
+        echo Finished downloading %WRAPPER_JAR%
+    )
+)
+@REM End of extension
+
+@REM Provide a "standardized" way to retrieve the CLI args that will
+@REM work with both Windows and non-Windows executions.
+set MAVEN_CMD_LINE_ARGS=%*
+
+%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
+if ERRORLEVEL 1 goto error
+goto end
+
+:error
+set ERROR_CODE=1
+
+:end
+@endlocal & set ERROR_CODE=%ERROR_CODE%
+
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
+@REM check for post script, once with legacy .bat ending and once with .cmd ending
+if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
+if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
+:skipRcPost
+
+@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
+if "%MAVEN_BATCH_PAUSE%" == "on" pause
+
+if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
+
+exit /B %ERROR_CODE%

pom.xml

@@ -0,0 +1,209 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.2.6.RELEASE</version>
+        <relativePath/> <!-- lookup parent from repository -->
+    </parent>
+    <groupId>com.winhc</groupId>
+    <artifactId>winhc-bigdata-task</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <name>winhc-bigdata-task</name>
+    <description>Demo project for Spring Boot</description>
+    <properties>
+        <java.version>1.8</java.version>
+        <phoenix.client.version>5.2.1-HBase-2.x</phoenix.client.version>
+        <spring.boot.version>2.0.3.RELEASE</spring.boot.version>
+        <mybatis.starter.version>1.2.0</mybatis.starter.version>
+        <mybatis-plus.version>3.4.3</mybatis-plus.version>
+        <druid.version>1.1.5</druid.version>
+        <sdk.version>0.33.7-public</sdk.version>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-aop</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-configuration-processor</artifactId>
+            <optional>true</optional>
+        </dependency>
+
+        <dependency>
+            <groupId>cn.hutool</groupId>
+            <artifactId>hutool-all</artifactId>
+            <version>5.3.5</version>
+        </dependency>
+
+
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-validation</artifactId>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>27.0.1-jre</version>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>fastjson</artifactId>
+            <version>1.2.56</version>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.6</version>
+        </dependency>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>8.0.16</version>
+        </dependency>
+
+
+        <dependency>
+            <groupId>com.baomidou</groupId>
+            <artifactId>dynamic-datasource-spring-boot-starter</artifactId>
+            <version>3.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.baomidou</groupId>
+            <artifactId>mybatis-plus-annotation</artifactId>
+            <version>${mybatis-plus.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.baomidou</groupId>
+            <artifactId>mybatis-plus-boot-starter</artifactId>
+            <version>${mybatis-plus.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.baomidou</groupId>
+            <artifactId>mybatis-plus</artifactId>
+            <version>${mybatis-plus.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>okhttp</artifactId>
+            <version>3.10.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>logging-interceptor</artifactId>
+            <version>3.10.0</version>
+        </dependency>
+
+
+        <!--        <dependency>-->
+        <!--            <groupId>org.elasticsearch.client</groupId>-->
+        <!--            <artifactId>rest</artifactId>-->
+        <!--            <version>5.5.3</version>-->
+        <!--        </dependency>-->
+        <!-- <dependency>
+             <groupId>org.elasticsearch</groupId>
+             <artifactId>elasticsearch</artifactId>
+             <version>5.6.0</version>
+         </dependency>
+         <dependency>
+             <groupId>org.elasticsearch.client</groupId>
+             <artifactId>elasticsearch-rest-client</artifactId>
+             <version>5.6.0</version>
+         </dependency>
+         <dependency>
+             <groupId>org.elasticsearch.client</groupId>
+             <artifactId>elasticsearch-rest-high-level-client</artifactId>
+             <version>5.6.0</version>
+             <exclusions>
+                 <exclusion>
+                     <groupId>org.elasticsearch.client</groupId>
+                     <artifactId>elasticsearch-rest-client</artifactId>
+                 </exclusion>
+             </exclusions>
+         </dependency>-->
+
+
+        <dependency>
+            <groupId>com.aliyun</groupId>
+            <artifactId>aliyun-java-sdk-core</artifactId>
+            <version>4.0.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.aliyun</groupId>
+            <artifactId>aliyun-java-sdk-dataworks-public</artifactId>
+            <version>1.8.3</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.kafka</groupId>
+            <artifactId>spring-kafka</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-mongodb</artifactId>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/com.aliyun.odps/odps-sdk-core -->
+        <dependency>
+            <groupId>com.aliyun.odps</groupId>
+            <artifactId>odps-sdk-core</artifactId>
+            <version>0.37.9-public</version>
+        </dependency>
+
+
+    </dependencies>
+
+
+    <build>
+        <finalName>winhc-bigdata-pull-task</finalName>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

src/main/java/com/winhc/bigdata/task/WinhcBigdataTaskApplication.java

@@ -0,0 +1,22 @@
+package com.winhc.bigdata.task;
+
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+/**
+ * @author x
+ */
+@EnableAsync
+@EnableScheduling
+@SpringBootApplication
+@MapperScan("com.winhc.bigdata.task.**.mapper")
+public class WinhcBigdataTaskApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(WinhcBigdataTaskApplication.class, args);
+    }
+
+}

src/main/java/com/winhc/bigdata/task/configuration/CORSConfiguration.java

@@ -0,0 +1,22 @@
+package com.winhc.bigdata.task.configuration;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.servlet.config.annotation.CorsRegistry;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/7 09:48
+ * @Description:
+ */
+@Configuration
+public class CORSConfiguration implements WebMvcConfigurer {
+    @Override
+    public void addCorsMappings(CorsRegistry registry) {
+        registry.addMapping("/**")
+                .allowedOrigins("*")
+                .allowCredentials(true)
+                .allowedMethods("*")
+                .maxAge(3600);
+    }
+}

src/main/java/com/winhc/bigdata/task/configuration/DataWorksAccessProperties.java

@@ -0,0 +1,21 @@
+package com.winhc.bigdata.task.configuration;
+
+import lombok.Data;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/6/8 15:56
+ * @Description:
+ */
+@Data
+@Configuration
+public class DataWorksAccessProperties {
+    @Value("${data-works.access-key-id}")
+    private String accessKeyId;
+    @Value("${data-works.access-key-secret}")
+    private String accessKeySecret;
+    @Value("${data-works.region-id}")
+    private String regionId;
+}

src/main/java/com/winhc/bigdata/task/configuration/DataWorksProjectConfiguration.java

@@ -0,0 +1,37 @@
+package com.winhc.bigdata.task.configuration;
+
+import com.aliyuncs.DefaultAcsClient;
+import com.aliyuncs.IAcsClient;
+import com.aliyuncs.profile.DefaultProfile;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/6/8 15:41
+ * @Description:
+ */
+@Configuration
+@Data
+@Slf4j
+@AllArgsConstructor
+public class DataWorksProjectConfiguration {
+    private final DataWorksAccessProperties dataWorksAccessProperties;
+
+    @SneakyThrows
+    @Bean
+    public IAcsClient client() {
+        DefaultProfile.addEndpoint(
+                "winhc-eci",
+                dataWorksAccessProperties.getRegionId(),
+                "dataworks-public",
+                "dataworks." + dataWorksAccessProperties.getRegionId() + ".aliyuncs.com");
+        DefaultProfile profile = DefaultProfile.getProfile(dataWorksAccessProperties.getRegionId(), dataWorksAccessProperties.getAccessKeyId(), dataWorksAccessProperties.getAccessKeySecret());
+        IAcsClient client = new DefaultAcsClient(profile);
+        return client;
+    }
+}

src/main/java/com/winhc/bigdata/task/configuration/ElasticSearchConfiguration.java

@@ -0,0 +1,91 @@
+//package com.winhc.bigdata.task.configuration;
+//
+//import org.apache.http.HttpHost;
+//import org.apache.http.auth.AuthScope;
+//import org.apache.http.auth.UsernamePasswordCredentials;
+//import org.apache.http.client.CredentialsProvider;
+//import org.apache.http.impl.client.BasicCredentialsProvider;
+//import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+//import org.elasticsearch.client.RestClient;
+//import org.elasticsearch.client.RestClientBuilder;
+//import org.elasticsearch.client.RestHighLevelClient;
+//import org.springframework.beans.factory.annotation.Value;
+//import org.springframework.context.annotation.Bean;
+//import org.springframework.context.annotation.Configuration;
+//
+//import java.util.stream.Stream;
+//
+///**
+// * @Author: XuJiakai
+// * @Date: 2020/9/15 20:01
+// * @Description:
+// */
+//@Configuration
+//public class ElasticSearchConfiguration {
+//    @Value("${es.username}")
+//    private String username;
+//    @Value("${es.password}")
+//    private String password;
+//    @Value("${es.host}")
+//    private String host;
+//
+//    @Value("${es.schema:http}")
+//    String schema;
+//    @Value(value = "${es.connect-timeout:100000}")
+//    String connectTimeout;
+//    @Value(value = "${es.socket-timeout:600000}")
+//    String socketTimeout;
+//    @Value(value = "${es.connection-request-timeout:50000}")
+//    String connectionRequestTimeout;
+//    @Value(value = "${es.max-conn-total:100}")
+//    String maxConnTotal;
+//    @Value(value = "${es.max-conn-per-route:100}")
+//    String maxConnPerRoute;
+//
+//    @Bean
+//    public RestClient bean() {
+//        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+//        credentialsProvider.setCredentials(AuthScope.ANY,
+//                new UsernamePasswordCredentials(username, password));
+//        // 单击所创建的Elasticsearch实例ID，在基本信息页面获取公网地址，即为HOST。
+//        return RestClient.builder(new HttpHost(host, 9200))
+//                .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+//                    @Override
+//                    public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+//                        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+//                    }
+//                }).build();
+//    }
+//
+//
+//    @Bean
+//    public RestHighLevelClient getClient() {
+//        HttpHost[] httpHosts = Stream.of(host.split(",")).map(host -> {
+//            String[] split = host.split(":");
+//            return new HttpHost(split[0], 9200, schema);
+//        }).toArray(HttpHost[]::new);
+//
+//        // 阿里云Elasticsearch集群需要basic auth验证。
+//        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+//        //访问用户名和密码为您创建阿里云Elasticsearch实例时设置的用户名和密码，也是Kibana控制台的登录用户名和密码。
+//        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
+//
+//
+//        return new RestHighLevelClient(RestClient
+//                .builder(httpHosts)
+//                .setMaxRetryTimeoutMillis(60000 * 3)
+//                .setRequestConfigCallback(builder -> {
+//                    builder.setConnectTimeout(!connectTimeout.isEmpty() ? Integer.valueOf(connectTimeout) : 1000);
+//                    builder.setSocketTimeout(!socketTimeout.isEmpty() ? Integer.valueOf(socketTimeout) : 60000);
+//                    builder.setConnectionRequestTimeout(!connectionRequestTimeout.isEmpty() ? Integer.valueOf(connectionRequestTimeout) : 500);
+//                    return builder;
+//                })
+//                .setHttpClientConfigCallback(httpAsyncClientBuilder -> {
+//                    httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+//                    httpAsyncClientBuilder.setMaxConnTotal(!maxConnTotal.isEmpty() ? Integer.valueOf(maxConnTotal) : 100);
+//                    httpAsyncClientBuilder.setMaxConnPerRoute(!maxConnPerRoute.isEmpty() ? Integer.valueOf(maxConnPerRoute) : 100);
+//                    return httpAsyncClientBuilder;
+//                }).build()
+//        );
+//    }
+//}

src/main/java/com/winhc/bigdata/task/configuration/KafkaConfiguration.java

@@ -0,0 +1,111 @@
+package com.winhc.bigdata.task.configuration;
+
+import com.google.common.collect.Maps;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.kafka.annotation.EnableKafka;
+import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
+import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
+import org.springframework.kafka.core.DefaultKafkaProducerFactory;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.core.ProducerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author: XuJiakai
+ * 2021/6/18 16:20
+ */
+@Configuration
+@EnableKafka
+public class KafkaConfiguration {
+
+    @Value("${spring.kafka.bootstrap-servers}")
+    private String bootstrapServers;
+
+    @Value("${spring.kafka.consumer.enable-auto-commit}")
+    private Boolean autoCommit;
+
+    @Value("${spring.kafka.producer.retries}")
+    private Integer retries;
+
+    @Value("${spring.kafka.producer.batch-size}")
+    private Integer batchSize;
+
+    @Value("${spring.kafka.producer.buffer-memory}")
+    private Integer bufferMemory;
+
+    /**
+     * 生产者配置信息
+     */
+    @Bean
+    public Map<String, Object> producerConfigs() {
+        Map<String, Object> props = Maps.newHashMap();
+        props.put(ProducerConfig.ACKS_CONFIG, "-1");
+        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+        props.put(ProducerConfig.RETRIES_CONFIG, retries);
+        props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
+        props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
+        props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
+        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+        props.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, "3000");
+        return props;
+    }
+
+    /**
+     * 生产者工厂
+     */
+    @Bean
+    public ProducerFactory<String, String> producerFactory() {
+        return new DefaultKafkaProducerFactory<>(producerConfigs());
+    }
+
+    /**
+     * 生产者模板
+     */
+    @Bean
+    public KafkaTemplate<String, String> kafkaTemplate() {
+        return new KafkaTemplate<>(producerFactory());
+    }
+
+    @Bean("containerFactory")
+    public ConcurrentKafkaListenerContainerFactory<String, String> containerFactory() {
+        ConcurrentKafkaListenerContainerFactory<String, String> container = new ConcurrentKafkaListenerContainerFactory<>();
+        container.setConsumerFactory(new DefaultKafkaConsumerFactory<>(consumerProps()));
+        // 设置并发量，小于或等于Topic的分区数
+        container.setConcurrency(1);
+        // 拉取超时时间
+        //container.getContainerProperties().setPollTimeout(1500);
+        // 设置为批量监听
+        container.setBatchListener(true);
+        // 设置提交偏移量的方式
+        //container.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
+        return container;
+    }
+
+    private Map<String, Object> consumerProps() {
+        Map<String, Object> props = new HashMap<>(8);
+        // kafka服务地址
+        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+        // 设置是否自动提交
+        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit);
+        // 一次拉取消息数量
+        props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1000);
+        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 5*60*1000);
+        props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5*60*1000);
+        // 最大处理时间
+        props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 5*60*1000);
+        // 序列化
+        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
+        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
+        return props;
+    }
+
+}

src/main/java/com/winhc/bigdata/task/configuration/MybatisPlusConfig.java

@@ -0,0 +1,27 @@
+package com.winhc.bigdata.task.configuration;
+
+import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
+import com.baomidou.mybatisplus.extension.plugins.inner.OptimisticLockerInnerInterceptor;
+import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/7 09:49
+ * @Description:
+ */
+@EnableTransactionManagement
+@Configuration
+public class MybatisPlusConfig {
+
+    @Bean
+    public MybatisPlusInterceptor mybatisPlusInterceptor() {
+        MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
+        interceptor.addInnerInterceptor(new PaginationInnerInterceptor());
+        interceptor.addInnerInterceptor(new OptimisticLockerInnerInterceptor());
+        return interceptor;
+    }
+}

src/main/java/com/winhc/bigdata/task/configuration/OdpsConfiguration.java

@@ -0,0 +1,36 @@
+package com.winhc.bigdata.task.configuration;
+
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.account.Account;
+import com.aliyun.odps.account.AliyunAccount;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/1 15:49
+ */
+@Configuration
+public class OdpsConfiguration {
+    @Value("${odps.access-key-id}")
+    private String accessKeyId;
+
+    @Value("${odps.access-key-secret}")
+    private String accessKeySecret;
+
+    @Value("${odps.default-project}")
+    private String defaultProject;
+
+    @Value("${odps.end-point}")
+    private String endPoint;
+
+    @Bean
+    public Odps odps() {
+        Account account = new AliyunAccount(accessKeyId, accessKeySecret);
+        Odps odps = new Odps(account);
+        odps.setEndpoint(endPoint);
+        odps.setDefaultProject(defaultProject);
+        return odps;
+    }
+}

src/main/java/com/winhc/bigdata/task/configuration/OkHttpConfiguration.java

@@ -0,0 +1,83 @@
+package com.winhc.bigdata.task.configuration;
+
+
+import lombok.extern.slf4j.Slf4j;
+import okhttp3.ConnectionPool;
+import okhttp3.OkHttpClient;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/6/28 14:37
+ * @Description:
+ */
+@Slf4j
+@Configuration
+public class OkHttpConfiguration {
+
+    @Bean
+    public X509TrustManager x509TrustManager() {
+        return new X509TrustManager() {
+            @Override
+            public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
+            }
+
+            @Override
+            public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
+            }
+
+            @Override
+            public X509Certificate[] getAcceptedIssuers() {
+                return new X509Certificate[0];
+            }
+        };
+    }
+
+    @Bean
+    public SSLSocketFactory sslSocketFactory() {
+        try {
+            //信任任何链接
+            SSLContext sslContext = SSLContext.getInstance("TLS");
+            sslContext.init(null, new TrustManager[]{x509TrustManager()}, new SecureRandom());
+            return sslContext.getSocketFactory();
+        } catch (NoSuchAlgorithmException | KeyManagementException e) {
+            log.error(e.getMessage(), e);
+        }
+        return null;
+    }
+
+    /**
+     * Create a new connection pool with tuning parameters appropriate for a single-user application.
+     * The tuning parameters in this pool are subject to change in future OkHttp releases. Currently
+     */
+    @Bean
+    public ConnectionPool pool() {
+        return new ConnectionPool(200, 5, TimeUnit.MINUTES);
+    }
+
+    @Bean
+    public OkHttpClient okHttpClient() {
+        return new OkHttpClient.Builder()
+                .sslSocketFactory(sslSocketFactory(), x509TrustManager())
+                .addInterceptor(new OkHttpRetryInterceptor(3))
+                //是否开启缓存
+                .retryOnConnectionFailure(false)
+                .connectionPool(pool())
+                .connectTimeout(10L, TimeUnit.SECONDS)
+                .readTimeout(10L, TimeUnit.SECONDS)
+                .build();
+    }
+
+}

src/main/java/com/winhc/bigdata/task/configuration/OkHttpRetryInterceptor.java

@@ -0,0 +1,48 @@
+package com.winhc.bigdata.task.configuration;
+
+import lombok.extern.slf4j.Slf4j;
+import okhttp3.Interceptor;
+import okhttp3.Request;
+import okhttp3.Response;
+
+import java.io.IOException;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/6/28 14:38
+ * @Description:
+ */
+@Slf4j
+public class OkHttpRetryInterceptor implements Interceptor {
+    //最大重试次数
+    public int maxRetry;
+    //假如设置为3次重试的话，则最大可能请求4次（默认1次+3次重试）
+
+    public OkHttpRetryInterceptor(int maxRetry) {
+        this.maxRetry = maxRetry;
+    }
+
+    @Override
+    public Response intercept(Chain chain) throws IOException {
+        Request request = chain.request();
+        int retryNum = 0;
+        while (true) {
+            try {
+                Response response = chain.proceed(request);
+                while (!response.isSuccessful() && retryNum < maxRetry) {
+                    retryNum++;
+                    log.error("response code : {} ok http retry:{} request url: {} ,response body:{}", response.code(), retryNum, request.url(), response.body().string());
+                    response.close();
+                    response = chain.proceed(request);
+                }
+                return response;
+            } catch (Exception e) {
+                log.error(e.getMessage(), e);
+                log.error("ok http retry:{} request url: {} ", retryNum, request.url());
+                if (retryNum++ > maxRetry) {
+                    throw e;
+                }
+            }
+        }
+    }
+}

src/main/java/com/winhc/bigdata/task/configuration/ProjectParamInit.java

@@ -0,0 +1,47 @@
+package com.winhc.bigdata.task.configuration;
+
+import com.winhc.bigdata.task.util.DataWorksParamUtils;
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowJob;
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowTask;
+import lombok.Getter;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
+import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.List;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/14 14:42
+ */
+@Getter
+@Configuration
+public class ProjectParamInit implements BeanFactoryPostProcessor {
+
+    public static final String project = "winhc_ng";
+
+    public static final String exportFlowName = "pull_data_export_data_flow";
+    public static final String exportBeanName = "winhc_ng:pull_data_export_data_flow:export_data";
+    public static final String generateFlowName = "pull_data_generate_data_flow";
+    public static final String generateBeanName = "winhc_ng:pull_data_generate_data_flow:generate_data";
+
+    /**
+     * @param beanFactory
+     * @throws BeansException
+     */
+    @Override
+    public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory) throws BeansException {
+        DataWorksParamUtils bean = new DataWorksParamUtils();
+        List<DataWorksFlowJob> jobs = bean.getJobs();
+        for (DataWorksFlowJob job : jobs) {
+            String project = job.getProject();
+            String flow = job.getFlow();
+            List<DataWorksFlowTask> task = job.getTask();
+            for (DataWorksFlowTask dataWorksFlowTask : task) {
+                String beanName = project + ":" + flow + ":" + dataWorksFlowTask.getTaskName();
+                beanFactory.registerSingleton(beanName, dataWorksFlowTask);
+            }
+        }
+    }
+}

src/main/java/com/winhc/bigdata/task/configuration/PropertiesListener.java

@@ -0,0 +1,22 @@
+package com.winhc.bigdata.task.configuration;
+
+import org.springframework.boot.context.event.ApplicationStartedEvent;
+import org.springframework.context.ApplicationListener;
+
+/**
+ * @author: XuJiakai
+ * 2021/6/18 18:37
+ */
+public class PropertiesListener implements ApplicationListener<ApplicationStartedEvent> {
+
+    private String propertyFileName;
+
+    public PropertiesListener(String propertyFileName) {
+        this.propertyFileName = propertyFileName;
+    }
+
+    @Override
+    public void onApplicationEvent(ApplicationStartedEvent event) {
+        PropertiesListenerConfig.loadAllProperties(propertyFileName);
+    }
+}

src/main/java/com/winhc/bigdata/task/configuration/PropertiesListenerConfig.java

@@ -0,0 +1,51 @@
+package com.winhc.bigdata.task.configuration;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeansException;
+import org.springframework.core.io.support.PropertiesLoaderUtils;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * @author: XuJiakai
+ * 2021/6/18 18:39
+ */
+@Slf4j
+public class PropertiesListenerConfig {
+    public static Map<String, String> propertiesMap = new HashMap();
+
+    private static void processProperties(Properties props) throws BeansException {
+        propertiesMap = new HashMap<String, String>();
+        for (Object key : props.keySet()) {
+            String keyStr = key.toString();
+            try {
+                // PropertiesLoaderUtils的默认编码是ISO-8859-1,在这里转码一下
+                propertiesMap.put(keyStr, new String(props.getProperty(keyStr).getBytes("ISO-8859-1"), "utf-8"));
+            } catch (Exception e) {
+                log.error(e.getMessage(),e);
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    public static void loadAllProperties(String propertyFileName) {
+        try {
+            Properties properties = PropertiesLoaderUtils.loadAllProperties(propertyFileName);
+            processProperties(properties);
+        } catch (IOException e) {
+            log.error(e.getMessage(),e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static String getProperty(String name) {
+        return propertiesMap.get(name).toString();
+    }
+
+    public static Map<String, String> getAllProperty() {
+        return propertiesMap;
+    }
+}

src/main/java/com/winhc/bigdata/task/configuration/ThreadPoolExecutorConfig.java

@@ -0,0 +1,53 @@
+package com.winhc.bigdata.task.configuration;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/14 15:24
+ */
+@Configuration
+public class ThreadPoolExecutorConfig {
+    /**
+     * 线程池维护线程的最少数量
+     */
+    @Value("${thread.pool.core.pool.size:1}")
+    private int threadPoolCorePoolSize;
+    /**
+     * 线程池维护线程的最大数量
+     */
+    @Value("${thread.pool.max.pool.size:1}")
+    private int threadPoolMaxPoolSize;
+    /**
+     * 缓存队列
+     */
+    @Value("${thread.pool.queue.capacity:10}")
+    private int threadPoolQueueCapacity;
+    /**
+     * 允许的空闲时间
+     */
+    @Value("${thread.pool.keep.alive.seconds:300}")
+    private int threadPoolKeepAliveSeconds;
+
+
+    @Bean(name = "export-data-pool")
+    public ThreadPoolTaskExecutor getMyThreadPoolExecutor() {
+        ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
+
+        taskExecutor.setCorePoolSize(threadPoolCorePoolSize);
+        taskExecutor.setMaxPoolSize(threadPoolMaxPoolSize);
+        taskExecutor.setQueueCapacity(threadPoolQueueCapacity);
+        taskExecutor.setKeepAliveSeconds(threadPoolKeepAliveSeconds);
+        //定义线程池名字
+        taskExecutor.setThreadNamePrefix("export-data-pool");
+        //对拒绝task的处理策略
+        taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
+        taskExecutor.initialize();
+        return taskExecutor;
+    }
+}

src/main/java/com/winhc/bigdata/task/configuration/ValidationAdvice.java

@@ -0,0 +1,68 @@
+package com.winhc.bigdata.task.configuration;
+
+import com.winhc.bigdata.task.service.DingTalkService;
+import com.winhc.bigdata.task.vo.ResponseVo;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.Ordered;
+import org.springframework.core.annotation.Order;
+import org.springframework.web.HttpRequestMethodNotSupportedException;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+import javax.validation.ConstraintViolationException;
+
+import static org.springframework.http.HttpStatus.BAD_REQUEST;
+import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/7 11:03
+ * @Description:
+ */
+@Slf4j
+@Order(Ordered.HIGHEST_PRECEDENCE)
+@ControllerAdvice
+@AllArgsConstructor
+public class ValidationAdvice {
+    private final DingTalkService dingTalkService;
+
+    @ResponseStatus(BAD_REQUEST)
+    @ResponseBody
+    @ExceptionHandler(ConstraintViolationException.class)
+    public ResponseVo<String> methodArgumentNotValidException(ConstraintViolationException ex) {
+        log.error(ex.getMessage(), ex);
+        dingTalkService.error(ex.getMessage());
+        return ResponseVo.failure(ex.getMessage());
+    }
+
+
+    @ResponseStatus(BAD_REQUEST)
+    @ResponseBody
+    @ExceptionHandler(IllegalArgumentException.class)
+    public ResponseVo<String> illegalArgumentException(IllegalArgumentException ex) {
+        log.error(ex.getMessage(), ex);
+        dingTalkService.error(ex.getMessage());
+        return ResponseVo.failure(ex.getMessage());
+    }
+
+    @ResponseStatus(BAD_REQUEST)
+    @ResponseBody
+    @ExceptionHandler(HttpRequestMethodNotSupportedException.class)
+    public ResponseVo<String> httpRequestMethodNotSupportedException(HttpRequestMethodNotSupportedException ex) {
+        log.error(ex.getMessage(), ex);
+        return ResponseVo.failure(ex.getMessage());
+    }
+
+
+    @ResponseStatus(INTERNAL_SERVER_ERROR)
+    @ResponseBody
+    @ExceptionHandler(Exception.class)
+    public ResponseVo<String> exception(Exception ex) {
+        log.error(ex.getMessage(), ex);
+        dingTalkService.error(ex.getMessage());
+        return ResponseVo.failure(ex.getMessage());
+    }
+}

src/main/java/com/winhc/bigdata/task/constant/DingTalkConst.java

@@ -0,0 +1,28 @@
+package com.winhc.bigdata.task.constant;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/12 14:07
+ */
+
+public enum DingTalkConst {
+    /**
+     * winhc日志报警通知群
+     */
+    winhc_log_talk("SECe7b26876f443e77f872b8b10880e39b3c5dfaf44855f1aa3235372bb73698ab6", "2773b742b74d84599c4f05f7b42cacd6714b10c33cd4c74402649019fa7e56c8");
+    private String dingSecret;
+    private String accessToken;
+
+    DingTalkConst(String dingSecret, String accessToken) {
+        this.dingSecret = dingSecret;
+        this.accessToken = accessToken;
+    }
+
+    public String getDingSecret() {
+        return dingSecret;
+    }
+
+    public String getAccessToken() {
+        return accessToken;
+    }
+}

src/main/java/com/winhc/bigdata/task/constant/SystemParams.java

@@ -0,0 +1,9 @@
+package com.winhc.bigdata.task.constant;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/20 11:02
+ */
+public class SystemParams {
+    public static final String sign = "hnjmkladfsvhjklnmadsvfhjklmnasdfvgbkln";
+}

src/main/java/com/winhc/bigdata/task/controller/MaxComputeCallBackController.java

@@ -0,0 +1,30 @@
+package com.winhc.bigdata.task.controller;
+
+import cn.hutool.core.lang.Assert;
+import com.winhc.bigdata.task.constant.SystemParams;
+import com.winhc.bigdata.task.service.PullDataService;
+import com.winhc.bigdata.task.vo.TablePartitionInfoVO;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.List;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/12 13:42
+ */
+@Slf4j
+@RestController
+@AllArgsConstructor
+@RequestMapping("call-back")
+public class MaxComputeCallBackController {
+    private final PullDataService pullDataService;
+
+    @PostMapping
+    public Object callBack(@RequestParam(required = false) String sign, @RequestBody List<TablePartitionInfoVO> tab) {
+        Assert.isTrue(SystemParams.sign.equals(sign), "验签不通过！");
+        log.info("max compute数据回调：{}", tab);
+        return pullDataService.callBack(tab);
+    }
+}

src/main/java/com/winhc/bigdata/task/controller/OperationsCenterController.java

@@ -0,0 +1,51 @@
+package com.winhc.bigdata.task.controller;
+
+import cn.hutool.core.util.RandomUtil;
+import com.winhc.bigdata.task.service.DataWorksService;
+import com.winhc.bigdata.task.service.DingTalkService;
+import com.winhc.bigdata.task.util.DateUtils;
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowTask;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.Map;
+
+/**
+ * @author: XuJiakai
+ * 2021/5/24 14:37
+ */
+@Slf4j
+@RestController
+@AllArgsConstructor
+@RequestMapping("operations-center")
+public class OperationsCenterController {
+    private final DataWorksService dataWorksService;
+    private final Map<String, DataWorksFlowTask> map;
+    private final DingTalkService dingTalkService;
+
+    @PostMapping("{project}/{flow}/{task}")
+    public Object touch(@PathVariable String project, @PathVariable String flow, @PathVariable String task, @RequestBody Map<String, String> m) {
+        String beanName = project + ":" + flow + ":" + task;
+        String yesterday = DateUtils.getYesterday();
+        DataWorksFlowTask dataWorksFlowTask = map.get(beanName);
+        if (!m.containsKey("target_tab_prefix")) {
+            m.put("target_tab_prefix", "eci_execution_person_data");
+        }
+        if (!m.containsKey("ds")) {
+            m.put("ds", yesterday + "-" + RandomUtil.randomString(8));
+        }
+
+        if (!m.containsKey("deleted")) {
+            m.put("deleted", "0");
+        }
+        Long touch = dataWorksService.touch(project, flow, yesterday, dataWorksFlowTask, m);
+        dingTalkService.info("触发运维中心手动业务流程：{}  dag-id：{}; request body:{}", beanName, touch,m);
+        return touch;
+    }
+
+    @GetMapping("{project}/{dag-id}")
+    public Object query(@PathVariable String project, @PathVariable("dag-id") Long dagId) {
+        return dataWorksService.query(project, dagId);
+    }
+}

src/main/java/com/winhc/bigdata/task/controller/PullDataController.java

@@ -0,0 +1,38 @@
+package com.winhc.bigdata.task.controller;
+
+import cn.hutool.core.lang.Assert;
+import com.winhc.bigdata.task.constant.SystemParams;
+import com.winhc.bigdata.task.framework.odps.service.PullRequestRecordService;
+import com.winhc.bigdata.task.service.PullDataService;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/12 11:49
+ */
+@Slf4j
+@RestController
+@AllArgsConstructor
+@RequestMapping("pull")
+public class PullDataController {
+    private final PullDataService pullDataService;
+    private final PullRequestRecordService pullRequestRecordService;
+
+    @GetMapping
+    public Object get(@RequestParam(required = false) String sign) {
+        Assert.isTrue(SystemParams.sign.equals(sign), "验签不通过！");
+        return pullDataService.pull();
+    }
+
+
+    @GetMapping("sql")
+    public Object sql(@RequestParam(required = false) String sign, String sqlId, String pullBy) {
+        Assert.isTrue(SystemParams.sign.equals(sign), "验签不通过！");
+        return pullRequestRecordService.pull(sqlId, pullBy);
+    }
+}

src/main/java/com/winhc/bigdata/task/controller/SqlController.java

@@ -0,0 +1,37 @@
+package com.winhc.bigdata.task.controller;
+
+import com.winhc.bigdata.task.framework.odps.service.OdpsSqlInfoService;
+import com.winhc.bigdata.task.framework.odps.vo.OdpsSqlInfoVO;
+import com.winhc.bigdata.task.framework.odps.vo.RegisterSqlVO;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.bind.annotation.*;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 09:42
+ */
+@Slf4j
+@RestController
+@RequestMapping("sql")
+@AllArgsConstructor
+public class SqlController {
+    private final OdpsSqlInfoService odpsSqlInfoService;
+
+
+    @PostMapping("register")
+    public Object registerSql(@RequestBody RegisterSqlVO registerSqlVO) {
+        return odpsSqlInfoService.registerSql(registerSqlVO);
+    }
+
+
+    @PostMapping("save")
+    public Object save(@RequestBody OdpsSqlInfoVO body) {
+        return odpsSqlInfoService.add(body);
+    }
+
+    @GetMapping
+    public Object get(@RequestParam String sqlId) {
+        return odpsSqlInfoService.get(sqlId);
+    }
+}

src/main/java/com/winhc/bigdata/task/controller/TestController.java

@@ -0,0 +1,61 @@
+package com.winhc.bigdata.task.controller;
+
+import com.aliyun.odps.OdpsException;
+import com.winhc.bigdata.task.framework.odps.ExportDataCenter;
+import com.winhc.bigdata.task.framework.odps.service.PullRequestRecordService;
+import com.winhc.bigdata.task.framework.odps.utils.ExecOdpsSqlUtil;
+import com.winhc.bigdata.task.framework.odps.vo.MaxComputeSqlInstance;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 11:48
+ */
+@Slf4j
+@RestController
+@RequestMapping("test")
+@AllArgsConstructor
+public class TestController {
+
+    private final PullRequestRecordService pullRequestRecordService;
+    private final ExportDataCenter exportDataCenter;
+    private final ExecOdpsSqlUtil execOdpsSqlUtil;
+
+    @GetMapping("pull")
+    public Object pull(String sqlId, String pullBy) {
+        return pullRequestRecordService.pull(sqlId, pullBy);
+    }
+
+
+    @GetMapping("test-pull")
+    public Object testPull() {
+        exportDataCenter.pull();
+        return true;
+    }
+
+    @GetMapping("test-sql")
+    public Object testSql() {
+        exportDataCenter.sql();
+        return true;
+    }
+
+    @GetMapping("test-export")
+    public Object testExport() {
+        exportDataCenter.export();
+        return true;
+    }
+
+    @GetMapping("instance-status")
+    public Object instanceStatus(String instanceId) {
+        try {
+            return MaxComputeSqlInstance.of(execOdpsSqlUtil.getInstance(instanceId));
+        } catch (OdpsException e) {
+            log.error(e.getMessage(), e);
+            return e.getMessage();
+        }
+    }
+}

src/main/java/com/winhc/bigdata/task/dao/CompanyIndexDao.java

@@ -0,0 +1,13 @@
+package com.winhc.bigdata.task.dao;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/13 14:25
+ */
+public interface CompanyIndexDao {
+    SearchResponse search(SearchSourceBuilder searchSourceBuilder);
+
+}

src/main/java/com/winhc/bigdata/task/dao/impl/CompanyIndexDaoImpl.java

@@ -0,0 +1,34 @@
+package com.winhc.bigdata.task.dao.impl;
+
+import com.winhc.bigdata.task.dao.CompanyIndexDao;
+import lombok.AllArgsConstructor;
+import lombok.SneakyThrows;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.springframework.stereotype.Repository;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/13 14:25
+ */
+@Repository
+@AllArgsConstructor
+public class CompanyIndexDaoImpl implements CompanyIndexDao {
+    private static final String index = "winhc-company-v8";
+    private static final String type = "company";
+    private final RestHighLevelClient restHighLevelClient;
+
+    @Override
+    @SneakyThrows
+    public SearchResponse search(SearchSourceBuilder searchSourceBuilder) {
+        SearchRequest searchRequest = new SearchRequest()
+                .indices(index)
+                .types(type)
+                .source(searchSourceBuilder);
+        return restHighLevelClient.search(searchRequest);
+    }
+
+
+}

src/main/java/com/winhc/bigdata/task/entity/MaxComputeDetailOutput.java

@@ -0,0 +1,104 @@
+package com.winhc.bigdata.task.entity;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableLogic;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.winhc.bigdata.task.enums.TaskStatusEnum;
+import com.winhc.bigdata.task.util.JsonUtils;
+import lombok.Builder;
+import lombok.Data;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * max compute输出表详细信息
+ *
+ * @author x
+ * @TableName max_compute_detail_output
+ */
+@Builder
+@TableName(value = "max_compute_detail_output")
+@Data
+public class MaxComputeDetailOutput implements Serializable {
+    /**
+     *
+     */
+    @TableId
+    private Long id;
+
+    /**
+     * pull data task 任务id
+     */
+    private String taskId;
+
+    private Long dagId;
+
+    /**
+     * max compute table name
+     */
+    private String maxComputeTableName;
+
+    /**
+     * 分区名
+     */
+    private String partitionName;
+
+    /**
+     * 该分区数量
+     */
+    private Long recordCount;
+
+    /**
+     * 优先级
+     */
+    private Integer priorityLevel;
+
+    /**
+     * 该分区任务状态
+     */
+    private TaskStatusEnum partitionTaskStatus;
+
+    /**
+     *
+     */
+    private Date updateTime;
+
+    /**
+     *
+     */
+    private Date createTime;
+
+    /**
+     *
+     */
+    @TableLogic
+    private Integer deleted;
+
+    @TableField(exist = false)
+    private static final long serialVersionUID = 112L;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+
+        if (o == null || getClass() != o.getClass()) return false;
+
+        MaxComputeDetailOutput that = (MaxComputeDetailOutput) o;
+
+        return new EqualsBuilder().append(id, that.id).append(taskId, that.taskId).append(dagId, that.dagId).append(maxComputeTableName, that.maxComputeTableName).append(partitionName, that.partitionName).append(recordCount, that.recordCount).append(priorityLevel, that.priorityLevel).append(partitionTaskStatus, that.partitionTaskStatus).append(updateTime, that.updateTime).append(createTime, that.createTime).append(deleted, that.deleted).isEquals();
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder(17, 37).append(id).append(taskId).append(dagId).append(maxComputeTableName).append(partitionName).append(recordCount).append(priorityLevel).append(partitionTaskStatus).append(updateTime).append(createTime).append(deleted).toHashCode();
+    }
+
+    @Override
+    public String toString() {
+        return JsonUtils.jsonObjToString(this);
+    }
+}

src/main/java/com/winhc/bigdata/task/entity/PullDataLog.java

@@ -0,0 +1,87 @@
+package com.winhc.bigdata.task.entity;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableLogic;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.winhc.bigdata.task.enums.TaskStatusEnum;
+import com.winhc.bigdata.task.util.JsonUtils;
+import lombok.Builder;
+import lombok.Data;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * null
+ *
+ * @author x
+ * @TableName pull_data_log
+ */
+@TableName(value = "pull_data_log")
+@Data
+@Builder
+public class PullDataLog implements Serializable {
+    /**
+     *
+     */
+    @TableId
+    private Long id;
+
+    /**
+     *
+     */
+    private Date requestTime;
+
+    /**
+     *
+     */
+    private String pullBy;
+
+    /**
+     *
+     */
+    private TaskStatusEnum pullStatus;
+
+    /**
+     * 创建时间
+     */
+    private Date createTime;
+
+    /**
+     * 更新时间
+     */
+    private Date updateTime;
+
+    /**
+     *
+     */
+    @TableLogic
+    private Integer deleted;
+
+    @TableField(exist = false)
+    private static final long serialVersionUID = 1L;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+
+        if (o == null || getClass() != o.getClass()) return false;
+
+        PullDataLog that = (PullDataLog) o;
+
+        return new EqualsBuilder().append(id, that.id).append(requestTime, that.requestTime).append(pullBy, that.pullBy).append(pullStatus, that.pullStatus).append(createTime, that.createTime).append(updateTime, that.updateTime).append(deleted, that.deleted).isEquals();
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder(17, 37).append(id).append(requestTime).append(pullBy).append(pullStatus).append(createTime).append(updateTime).append(deleted).toHashCode();
+    }
+
+    @Override
+    public String toString() {
+        return JsonUtils.jsonObjToString(this);
+    }
+}

src/main/java/com/winhc/bigdata/task/entity/PullDataTask.java

@@ -0,0 +1,91 @@
+package com.winhc.bigdata.task.entity;
+
+import com.baomidou.mybatisplus.annotation.*;
+import com.winhc.bigdata.task.enums.TaskStatusEnum;
+import com.winhc.bigdata.task.util.JsonUtils;
+import lombok.Builder;
+import lombok.Data;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * 拉数据任务主表
+ *
+ * @author x
+ * @TableName pull_data_task
+ */
+@TableName(value = "pull_data_task")
+@Data
+@Builder
+public class PullDataTask implements Serializable {
+    /**
+     *
+     */
+    @TableId(type = IdType.INPUT)
+    private String id;
+
+
+    private Long dagId;
+    /**
+     *
+     */
+    private String maxComputeFlowName;
+
+    /**
+     *
+     */
+    private Date startTime;
+
+    /**
+     *
+     */
+    private Date endTime;
+
+    /**
+     *
+     */
+    private TaskStatusEnum taskStatus;
+
+    /**
+     *
+     */
+    private Date createTime;
+
+    /**
+     *
+     */
+    private Date updateTime;
+
+    /**
+     *
+     */
+    @TableLogic
+    private Integer deleted;
+
+    @TableField(exist = false)
+    private static final long serialVersionUID = 1L;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+
+        if (o == null || getClass() != o.getClass()) return false;
+
+        PullDataTask that = (PullDataTask) o;
+
+        return new EqualsBuilder().append(id, that.id).append(dagId, that.dagId).append(maxComputeFlowName, that.maxComputeFlowName).append(startTime, that.startTime).append(endTime, that.endTime).append(taskStatus, that.taskStatus).append(createTime, that.createTime).append(updateTime, that.updateTime).append(deleted, that.deleted).isEquals();
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder(17, 37).append(id).append(dagId).append(maxComputeFlowName).append(startTime).append(endTime).append(taskStatus).append(createTime).append(updateTime).append(deleted).toHashCode();
+    }
+
+    @Override
+    public String toString() {
+        return JsonUtils.jsonObjToString(this);
+    }
+}

src/main/java/com/winhc/bigdata/task/enums/DataWorksTaskFlowEnum.java

@@ -0,0 +1,43 @@
+package com.winhc.bigdata.task.enums;
+
+import lombok.Getter;
+
+import java.util.Arrays;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/6/23 10:58
+ * @Description:
+ */
+@Getter
+@SuppressWarnings("all")
+public enum DataWorksTaskFlowEnum {
+    NOT_RUN(1, "未运行"),
+    WAIT_TIME(2, "等待时间"),
+    WAIT_RESOURCE(3, "等待资源"),
+    RUNNING(4, "运行中"),
+    FAILURE(5, "运行失败"),
+    SUCCESS(6, "运行成功"),
+    CHECKING(7, "校检中"),
+
+    ;
+    private final Integer code;
+    private final String msg;
+
+    DataWorksTaskFlowEnum(int code, String msg) {
+        this.code = code;
+        this.msg = msg;
+    }
+
+    public static DataWorksTaskFlowEnum getTaskFlowEnumByCode(Integer code) {
+        return Arrays.stream(DataWorksTaskFlowEnum.values())
+                .filter(taskFlowEnum -> taskFlowEnum.getCode().equals(code))
+                .findFirst()
+                .orElse(null);
+    }
+
+    @Override
+    public String toString() {
+        return msg;
+    }
+}

src/main/java/com/winhc/bigdata/task/enums/TaskStatusEnum.java

@@ -0,0 +1,24 @@
+package com.winhc.bigdata.task.enums;
+
+import com.baomidou.mybatisplus.annotation.EnumValue;
+import lombok.Getter;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/13 19:24
+ */
+@Getter
+@SuppressWarnings("all")
+public enum TaskStatusEnum {
+    生成中(1),
+    生成完毕(2),
+    导出中(3),
+    导出完毕(4),
+    等待中(5);
+    @EnumValue
+    private final int code;
+
+    TaskStatusEnum(int code) {
+        this.code = code;
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/AsyncTaskHandle.java

@@ -0,0 +1,43 @@
+package com.winhc.bigdata.task.framework.odps;
+
+import com.winhc.bigdata.task.framework.odps.exception.AsyncTaskRunningException;
+
+import java.util.Map;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/2 10:22
+ */
+public interface AsyncTaskHandle {
+
+
+    /**
+     * 启动任务
+     *
+     * @param taskId
+     * @param taskId
+     * @return
+     * @throws AsyncTaskRunningException 存在任务正在运行中，不能重复提交
+     */
+    String run(String taskId,String operatorName) throws AsyncTaskRunningException;
+
+
+    /**
+     * 判断并更新任务是否成功
+     *
+     * @param instanceId
+     * @return
+     */
+    Boolean isSuccess(String instanceId);
+
+    /**
+     * 回调，用于更新任务状态
+     *
+     * @param map
+     * @return
+     */
+    default Boolean callBack(Map<String, String> map) {
+        return true;
+    }
+
+}

src/main/java/com/winhc/bigdata/task/framework/odps/ExportDataCenter.java

@@ -0,0 +1,142 @@
+package com.winhc.bigdata.task.framework.odps;
+
+import com.winhc.bigdata.task.enums.DataWorksTaskFlowEnum;
+import com.winhc.bigdata.task.enums.TaskStatusEnum;
+import com.winhc.bigdata.task.framework.odps.entity.MaxComputeInstance;
+import com.winhc.bigdata.task.framework.odps.entity.OdpsResultInfo;
+import com.winhc.bigdata.task.framework.odps.entity.PullRequestRecord;
+import com.winhc.bigdata.task.framework.odps.exception.AsyncTaskRunningException;
+import com.winhc.bigdata.task.framework.odps.service.MaxComputeInstanceService;
+import com.winhc.bigdata.task.framework.odps.service.OdpsResultInfoService;
+import com.winhc.bigdata.task.framework.odps.service.PullRequestRecordService;
+import com.winhc.bigdata.task.framework.odps.utils.ExecOdpsSqlUtil;
+import com.winhc.bigdata.task.framework.odps.utils.ExportFlowUtils;
+import com.winhc.bigdata.task.service.DataWorksService;
+import com.winhc.bigdata.task.service.DingTalkService;
+import com.winhc.bigdata.task.util.DataWorksFlowStatusUtil;
+import lombok.AllArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 10:24
+ */
+@Slf4j
+@Component
+@AllArgsConstructor
+public class ExportDataCenter {
+    private final PullRequestRecordService pullRequestRecordService;
+    private final OdpsResultInfoService odpsResultInfoService;
+    private final MaxComputeInstanceService maxComputeInstanceService;
+    private final ExportFlowUtils exportFlowUtils;
+    private final ExecOdpsSqlUtil execOdpsSqlUtil;
+    private final DataWorksService dataWorksService;
+    private final DingTalkService dingTalkService;
+
+    private final AsyncTaskHandle asyncTaskHandle;
+
+    /**
+     * 定时判断是否有拉数据请求
+     */
+    @Scheduled(cron = "0 0/1 * * * ? ")
+    public synchronized void pull() {
+//        log.info("处理拉数据请求...");
+        List<PullRequestRecord> list = pullRequestRecordService.lambdaQuery()
+                .ne(PullRequestRecord::getPartitionTaskStatus, TaskStatusEnum.导出完毕)
+                .isNull(PullRequestRecord::getPullInstanceId)
+                .list();
+        for (PullRequestRecord record : list) {
+            String sqlId = record.getSqlId();
+            String operatorName = record.getPullBy();
+            log.info("拉取数据人：{} 拉取sqlId:{}", operatorName, sqlId);
+            pullDetail(sqlId, operatorName);
+        }
+    }
+
+
+    private void pullDetail(String sqlId, String operatorName) {
+        List<OdpsResultInfo> list = odpsResultInfoService.lambdaQuery()
+                .eq(OdpsResultInfo::getSqlId, sqlId)
+                .eq(OdpsResultInfo::getPartitionTaskStatus, TaskStatusEnum.生成完毕)
+                .orderByAsc(OdpsResultInfo::getPriorityLevel)
+                .list();
+        if (list.isEmpty()) {
+            //执行一个sql
+            try {
+                asyncTaskHandle.run(sqlId, operatorName);
+            } catch (AsyncTaskRunningException e) {
+            }
+        } else {
+            //启动一个导数据流程
+            OdpsResultInfo odpsResultInfo = list.get(0);
+            PullRequestRecord one = pullRequestRecordService.lambdaQuery()
+                    .eq(PullRequestRecord::getSqlId, sqlId)
+                    .isNull(PullRequestRecord::getPullInstanceId)
+                    .one();
+
+            String exportInstanceId = exportFlowUtils.export(odpsResultInfo.getPartitionName()) + "";
+
+            log.info("启动导数据任务：{} {}", odpsResultInfo.getTableName(), odpsResultInfo.getPartitionName());
+            one.setPullInstanceId(exportInstanceId);
+            one.setPartitionTaskStatus(TaskStatusEnum.导出中);
+            one.setOdpsResultInfoId(odpsResultInfo.getId());
+            pullRequestRecordService.updateById(one);
+        }
+    }
+
+
+    /**
+     * 定时判断sql是否执行完毕
+     */
+    @Scheduled(cron = "0 0/1 * * * ? ")
+    public synchronized void sql() {
+//        log.info("处理sql执行任务...");
+        List<MaxComputeInstance> list = maxComputeInstanceService.lambdaQuery()
+                .eq(MaxComputeInstance::getInstanceTerminated, false)
+                .list();
+        for (MaxComputeInstance instance : list) {
+            asyncTaskHandle.isSuccess(instance.getId());
+        }
+    }
+
+
+    /**
+     * 定时判断导数据DI是否完成
+     */
+    @Scheduled(cron = "0 0/1 * * * ? ")
+    @SneakyThrows
+    public synchronized void export() {
+//        log.info("处理导出数据任务...");
+        List<PullRequestRecord> list = pullRequestRecordService.lambdaQuery()
+                .ne(PullRequestRecord::getPartitionTaskStatus, TaskStatusEnum.导出完毕)
+                .isNotNull(PullRequestRecord::getPullInstanceId)
+                .list();
+
+        for (PullRequestRecord record : list) {
+            String pullInstanceId = record.getPullInstanceId();
+            Map<String, DataWorksTaskFlowEnum> winhc_ng = dataWorksService.query("winhc_ng", Long.parseLong(pullInstanceId));
+            Boolean successful = DataWorksFlowStatusUtil.isSuccessful(winhc_ng);
+
+            if (successful != null) {
+                if (successful) {
+                    record.setPartitionTaskStatus(TaskStatusEnum.导出完毕);
+                    odpsResultInfoService.lambdaUpdate()
+                            .eq(OdpsResultInfo::getId, record.getOdpsResultInfoId())
+                            .set(OdpsResultInfo::getPartitionTaskStatus, TaskStatusEnum.导出完毕)
+                            .update();
+                } else {
+                    log.error("导出失败");
+                    dingTalkService.error("导出数据失败，dagId:{}", pullInstanceId);
+                    record.setPartitionTaskStatus(TaskStatusEnum.等待中);
+                }
+                pullRequestRecordService.updateById(record);
+            }
+        }
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/entity/MaxComputeInstance.java

@@ -0,0 +1,50 @@
+package com.winhc.bigdata.task.framework.odps.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableLogic;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Builder;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/2 09:54
+ */
+@Builder
+@TableName(value = "max_compute_instance")
+@Data
+public class MaxComputeInstance implements Serializable {
+    @TableId(type = IdType.INPUT)
+    private String id;
+
+    private String sqlId;
+
+    private String type;
+
+    /**
+     * json 格式
+     */
+    private String detail;
+
+    private Boolean successful;
+
+    private Boolean instanceTerminated;
+    /**
+     *
+     */
+    private Date updateTime;
+
+    /**
+     *
+     */
+    private Date createTime;
+
+
+    @TableLogic
+    private Integer deleted;
+
+}

src/main/java/com/winhc/bigdata/task/framework/odps/entity/OdpsResultInfo.java

@@ -0,0 +1,71 @@
+package com.winhc.bigdata.task.framework.odps.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableLogic;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.winhc.bigdata.task.enums.TaskStatusEnum;
+import lombok.Builder;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 15:17
+ */
+@Builder
+@TableName(value = "odps_result_info")
+@Data
+public class OdpsResultInfo implements Serializable {
+    @TableId(type = IdType.AUTO)
+    private Long id;
+
+    private String sqlId;
+
+    private String instanceId;
+
+    private String instanceType;
+
+    private String tableName;
+
+    private String partitionName;
+
+    /**
+     * 该分区数量
+     */
+    private Long recordCount;
+
+    /**
+     * 优先级，数值越小优先级越高
+     */
+    private Integer priorityLevel;
+
+
+    /**
+     * json 格式
+     */
+    private String detail;
+
+
+    /**
+     * 该分区任务状态，生成中，生成完毕，导出完毕
+     */
+    private TaskStatusEnum partitionTaskStatus;
+
+    /**
+     *
+     */
+    private Date updateTime;
+
+    /**
+     *
+     */
+    private Date createTime;
+
+
+    @TableLogic
+    private Integer deleted;
+
+}

src/main/java/com/winhc/bigdata/task/framework/odps/entity/OdpsSqlInfo.java

@@ -0,0 +1,51 @@
+package com.winhc.bigdata.task.framework.odps.entity;
+
+import com.baomidou.mybatisplus.annotation.*;
+import lombok.Builder;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 15:43
+ */
+@Builder
+@TableName(value = "odps_sql_info")
+@Data
+public class OdpsSqlInfo implements Serializable {
+    @TableId(type = IdType.INPUT)
+    private String id;
+
+    private String sqlStr;
+
+    private String outputTab;
+
+    /**
+     * 分区数量
+     */
+    private Integer partitions;
+
+    private Boolean lockSign;
+
+    private String lockByName;
+
+    /**
+     *
+     */
+    private Date updateTime;
+
+    /**
+     *
+     */
+    private Date createTime;
+
+    private String createBy;
+
+    @Version
+    private Integer version;
+
+    @TableLogic
+    private Integer deleted;
+}

src/main/java/com/winhc/bigdata/task/framework/odps/entity/PullRequestRecord.java

@@ -0,0 +1,58 @@
+package com.winhc.bigdata.task.framework.odps.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableLogic;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.winhc.bigdata.task.enums.TaskStatusEnum;
+import lombok.Builder;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 10:06
+ */
+@Builder
+@TableName(value = "pull_request_record")
+@Data
+public class PullRequestRecord implements Serializable {
+    @TableId(type = IdType.AUTO)
+    private Long id;
+
+
+    private String sqlId;
+
+
+    private Long odpsResultInfoId;
+
+    /**
+     * 导出数据的实例id
+     */
+    private String pullInstanceId;
+
+    /**
+     * 任务状态，导出中，导出完毕，等待中
+     */
+    private TaskStatusEnum partitionTaskStatus;
+
+    private String pullBy;
+
+    /**
+     *
+     */
+    private Date updateTime;
+
+    /**
+     *
+     */
+    private Date createTime;
+
+
+    @TableLogic
+    private Integer deleted;
+
+
+}

src/main/java/com/winhc/bigdata/task/framework/odps/exception/AsyncTaskException.java

@@ -0,0 +1,27 @@
+package com.winhc.bigdata.task.framework.odps.exception;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/2 10:39
+ */
+public class AsyncTaskException extends RuntimeException{
+    public AsyncTaskException() {
+        super();
+    }
+
+    public AsyncTaskException(String message) {
+        super(message);
+    }
+
+    public AsyncTaskException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public AsyncTaskException(Throwable cause) {
+        super(cause);
+    }
+
+    protected AsyncTaskException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/exception/AsyncTaskRunningException.java

@@ -0,0 +1,27 @@
+package com.winhc.bigdata.task.framework.odps.exception;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/2 10:41
+ */
+public class AsyncTaskRunningException extends Exception{
+    public AsyncTaskRunningException() {
+        super();
+    }
+
+    public AsyncTaskRunningException(String message) {
+        super(message);
+    }
+
+    public AsyncTaskRunningException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public AsyncTaskRunningException(Throwable cause) {
+        super(cause);
+    }
+
+    protected AsyncTaskRunningException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/handle/SqlTaskHandle.java

@@ -0,0 +1,130 @@
+package com.winhc.bigdata.task.framework.odps.handle;
+
+import cn.hutool.core.util.StrUtil;
+import com.aliyun.odps.Instance;
+import com.winhc.bigdata.task.enums.TaskStatusEnum;
+import com.winhc.bigdata.task.framework.odps.AsyncTaskHandle;
+import com.winhc.bigdata.task.framework.odps.entity.MaxComputeInstance;
+import com.winhc.bigdata.task.framework.odps.entity.OdpsResultInfo;
+import com.winhc.bigdata.task.framework.odps.exception.AsyncTaskRunningException;
+import com.winhc.bigdata.task.framework.odps.mapper.MaxComputeInstanceMapper;
+import com.winhc.bigdata.task.framework.odps.service.OdpsResultInfoService;
+import com.winhc.bigdata.task.framework.odps.service.OdpsSqlInfoService;
+import com.winhc.bigdata.task.framework.odps.utils.ExecOdpsSqlUtil;
+import com.winhc.bigdata.task.framework.odps.vo.MaxComputeSqlInstance;
+import com.winhc.bigdata.task.framework.odps.vo.OdpsSqlInfoVO;
+import com.winhc.bigdata.task.util.GenerateIdUtils;
+import com.winhc.bigdata.task.util.JsonUtils;
+import lombok.AllArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 16:49
+ */
+@Slf4j
+@Service("sql_task")
+@AllArgsConstructor
+public class SqlTaskHandle implements AsyncTaskHandle {
+    private final OdpsSqlInfoService sqlInfoService;
+    private final ExecOdpsSqlUtil execOdpsSqlUtil;
+    private final MaxComputeInstanceMapper mapper;
+    private final OdpsResultInfoService odpsResultInfoService;
+
+    @Override
+    public String run(String taskId, String operatorName) throws AsyncTaskRunningException {
+        Boolean lock = sqlInfoService.lock(taskId, operatorName);
+        if (!lock) {
+            throw new AsyncTaskRunningException("存在运行中的任务");
+        }
+        try {
+            OdpsSqlInfoVO odpsSqlInfoVO = sqlInfoService.get(taskId);
+            HashMap<String, String> map = new HashMap<>(3);
+            String randomId = GenerateIdUtils.generateTaskId();
+            map.put("random", randomId);
+
+            String sql = StrUtil.format(odpsSqlInfoVO.getSql(), map);
+
+            MaxComputeSqlInstance exec = execOdpsSqlUtil.exec(sql);
+
+            MaxComputeInstance maxComputeInstance = MaxComputeInstance.builder()
+                    .id(exec.getId())
+                    .sqlId(taskId)
+                    .type("sql_task")
+                    .detail(JsonUtils.jsonObjToString(exec))
+                    .successful(false)
+                    .instanceTerminated(false)
+                    .build();
+            mapper.insert(maxComputeInstance);
+
+            List<OdpsResultInfo> list = new ArrayList<>();
+            for (Integer i = 1; i <= odpsSqlInfoVO.getPartitions(); i++) {
+                OdpsResultInfo sql_task = OdpsResultInfo.builder()
+                        .sqlId(taskId)
+                        .instanceId(exec.getId())
+                        .instanceType("sql_task")
+                        .tableName(odpsSqlInfoVO.getOutputTab())
+                        .partitionName(randomId + "_" + i)
+                        .recordCount(-1L)
+                        .priorityLevel(i)
+                        .partitionTaskStatus(TaskStatusEnum.生成中)
+                        .build();
+                list.add(sql_task);
+            }
+
+            odpsResultInfoService.saveBatch(list);
+            log.info("执行sql:{} 执行人：{}", taskId, operatorName);
+            return exec.getId();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @SneakyThrows
+    @Override
+    public Boolean isSuccess(String instanceId) {
+        MaxComputeInstance maxComputeInstance = mapper.selectById(instanceId);
+        if (maxComputeInstance.getInstanceTerminated()) {
+            return maxComputeInstance.getSuccessful();
+        }
+        Instance instance = execOdpsSqlUtil.getInstance(instanceId);
+        MaxComputeSqlInstance of = MaxComputeSqlInstance.of(instance);
+        if (of.getTerminated()) {
+
+
+            MaxComputeInstance task = MaxComputeInstance.builder()
+                    .id(of.getId())
+                    .sqlId(maxComputeInstance.getSqlId())
+                    .type("sql_task")
+                    .detail(JsonUtils.jsonObjToString(of))
+                    .successful(of.getSuccessful())
+                    .instanceTerminated(of.getTerminated())
+                    .build();
+            mapper.updateById(task);
+            // fixme 更新结果表
+            if (of.getSuccessful()) {
+                log.info("sqlId: {} 执行成功 ！", maxComputeInstance.getSqlId());
+                odpsResultInfoService.lambdaUpdate()
+                        .eq(OdpsResultInfo::getInstanceId, instanceId)
+                        .set(OdpsResultInfo::getPartitionTaskStatus, TaskStatusEnum.生成完毕)
+                        .update();
+            } else {
+                List<OdpsResultInfo> list = odpsResultInfoService.lambdaQuery().eq(OdpsResultInfo::getInstanceId, instanceId).list();
+                List<Long> collect = list.stream().map(OdpsResultInfo::getId).collect(Collectors.toList());
+                odpsResultInfoService.removeByIds(collect);
+                log.error("sqlId: {} 执行失败 ！", maxComputeInstance.getSqlId());
+            }
+            sqlInfoService.unlock(maxComputeInstance.getSqlId());
+            return of.getSuccessful();
+        } else {
+            return false;
+        }
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/mapper/MaxComputeInstanceMapper.java

@@ -0,0 +1,11 @@
+package com.winhc.bigdata.task.framework.odps.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.winhc.bigdata.task.framework.odps.entity.MaxComputeInstance;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/2 10:02
+ */
+public interface MaxComputeInstanceMapper extends BaseMapper<MaxComputeInstance> {
+}

src/main/java/com/winhc/bigdata/task/framework/odps/mapper/OdpsResultInfoMapper.java

@@ -0,0 +1,11 @@
+package com.winhc.bigdata.task.framework.odps.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.winhc.bigdata.task.framework.odps.entity.OdpsResultInfo;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 15:21
+ */
+public interface OdpsResultInfoMapper extends BaseMapper<OdpsResultInfo> {
+}

src/main/java/com/winhc/bigdata/task/framework/odps/mapper/OdpsSqlInfoMapper.java

@@ -0,0 +1,11 @@
+package com.winhc.bigdata.task.framework.odps.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.winhc.bigdata.task.framework.odps.entity.OdpsSqlInfo;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 16:17
+ */
+public interface OdpsSqlInfoMapper extends BaseMapper<OdpsSqlInfo> {
+}

src/main/java/com/winhc/bigdata/task/framework/odps/mapper/PullRequestRecordMapper.java

@@ -0,0 +1,11 @@
+package com.winhc.bigdata.task.framework.odps.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.winhc.bigdata.task.framework.odps.entity.PullRequestRecord;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 10:08
+ */
+public interface PullRequestRecordMapper extends BaseMapper<PullRequestRecord> {
+}

src/main/java/com/winhc/bigdata/task/framework/odps/service/MaxComputeInstanceService.java

@@ -0,0 +1,11 @@
+package com.winhc.bigdata.task.framework.odps.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import com.winhc.bigdata.task.framework.odps.entity.MaxComputeInstance;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 10:29
+ */
+public interface MaxComputeInstanceService extends IService< MaxComputeInstance> {
+}

src/main/java/com/winhc/bigdata/task/framework/odps/service/OdpsResultInfoService.java

@@ -0,0 +1,12 @@
+package com.winhc.bigdata.task.framework.odps.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import com.winhc.bigdata.task.framework.odps.entity.OdpsResultInfo;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 15:22
+ */
+public interface OdpsResultInfoService extends IService<OdpsResultInfo> {
+
+}

src/main/java/com/winhc/bigdata/task/framework/odps/service/OdpsSqlInfoService.java

@@ -0,0 +1,65 @@
+package com.winhc.bigdata.task.framework.odps.service;
+
+import com.winhc.bigdata.task.framework.odps.vo.OdpsSqlInfoVO;
+import com.winhc.bigdata.task.framework.odps.vo.RegisterSqlVO;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 16:18
+ */
+public interface OdpsSqlInfoService {
+
+
+    /**
+     * 为sql加锁
+     *
+     * @param sqlId
+     * @param lockByName
+     * @return false 上锁失败，已经被加过锁
+     */
+    Boolean lock(String sqlId, String lockByName);
+
+
+    /**
+     * 为sql解锁
+     *
+     * @param sqlId
+     * @return
+     */
+    Boolean unlock(String sqlId);
+
+
+    /**
+     * 快捷注册函数
+     *
+     * @param registerSqlVO
+     * @return
+     */
+    OdpsSqlInfoVO registerSql(RegisterSqlVO registerSqlVO);
+
+
+    /**
+     * 添加sql
+     *
+     * @param sql
+     * @return
+     */
+    String add(OdpsSqlInfoVO sql);
+
+    /**
+     * get
+     *
+     * @param sqlId
+     * @return
+     */
+    OdpsSqlInfoVO get(String sqlId);
+
+
+    /**
+     * check
+     *
+     * @param sqlId
+     * @return
+     */
+    Boolean check(String sqlId);
+}

src/main/java/com/winhc/bigdata/task/framework/odps/service/PullRequestRecordService.java

@@ -0,0 +1,21 @@
+package com.winhc.bigdata.task.framework.odps.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import com.winhc.bigdata.task.framework.odps.entity.PullRequestRecord;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 10:09
+ */
+public interface PullRequestRecordService extends IService<PullRequestRecord> {
+
+    /**
+     * 拉数据请求
+     *
+     * @param sqlId
+     * @param pullBy
+     * @return
+     */
+    Object pull(String sqlId, String pullBy);
+
+}

src/main/java/com/winhc/bigdata/task/framework/odps/service/impl/MaxComputeInstanceServiceImpl.java

@@ -0,0 +1,15 @@
+package com.winhc.bigdata.task.framework.odps.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.winhc.bigdata.task.framework.odps.entity.MaxComputeInstance;
+import com.winhc.bigdata.task.framework.odps.mapper.MaxComputeInstanceMapper;
+import com.winhc.bigdata.task.framework.odps.service.MaxComputeInstanceService;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 10:30
+ */
+@Service
+public class MaxComputeInstanceServiceImpl extends ServiceImpl<MaxComputeInstanceMapper, MaxComputeInstance> implements MaxComputeInstanceService {
+}

src/main/java/com/winhc/bigdata/task/framework/odps/service/impl/OdpsResultInfoServiceImpl.java

@@ -0,0 +1,15 @@
+package com.winhc.bigdata.task.framework.odps.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.winhc.bigdata.task.framework.odps.entity.OdpsResultInfo;
+import com.winhc.bigdata.task.framework.odps.service.OdpsResultInfoService;
+import org.springframework.stereotype.Service;
+import com.winhc.bigdata.task.framework.odps.mapper.OdpsResultInfoMapper;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 15:22
+ */
+@Service
+public class OdpsResultInfoServiceImpl extends ServiceImpl<OdpsResultInfoMapper, OdpsResultInfo> implements OdpsResultInfoService {
+}

src/main/java/com/winhc/bigdata/task/framework/odps/service/impl/OdpsSqlInfoServiceImpl.java

@@ -0,0 +1,154 @@
+package com.winhc.bigdata.task.framework.odps.service.impl;
+
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.crypto.digest.DigestUtil;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.winhc.bigdata.task.framework.odps.entity.OdpsSqlInfo;
+import com.winhc.bigdata.task.framework.odps.mapper.OdpsSqlInfoMapper;
+import com.winhc.bigdata.task.framework.odps.service.OdpsSqlInfoService;
+import com.winhc.bigdata.task.framework.odps.utils.OdpsSchemaUtils;
+import com.winhc.bigdata.task.framework.odps.vo.OdpsSqlInfoVO;
+import com.winhc.bigdata.task.framework.odps.vo.RegisterSqlVO;
+import com.winhc.bigdata.task.framework.odps.vo.SortVO;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+import org.springframework.util.Assert;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 16:20
+ */
+@Slf4j
+@Service
+@AllArgsConstructor
+public class OdpsSqlInfoServiceImpl extends ServiceImpl<OdpsSqlInfoMapper, OdpsSqlInfo> implements OdpsSqlInfoService {
+    private final OdpsSchemaUtils odpsSchemaUtils;
+
+    @Override
+    public Boolean lock(String sqlId, String lockByName) {
+        OdpsSqlInfo byId = getById(sqlId);
+        Assert.notNull(byId, "sql id 不存在！");
+        if (byId.getLockSign()) {
+            log.error("sqlId: {} lock failure !", sqlId);
+            return false;
+        } else {
+            byId.setLockSign(true);
+            byId.setLockByName(lockByName);
+            updateById(byId);
+            log.info("sqlId: {} lock successful !", sqlId);
+            return true;
+        }
+    }
+
+    @Override
+    public Boolean unlock(String sqlId) {
+        OdpsSqlInfo byId = getById(sqlId);
+        Assert.notNull(byId, "sql id 不存在！");
+        if (byId.getLockSign()) {
+            byId.setLockSign(false);
+            byId.setLockByName(null);
+            updateById(byId);
+            log.info("sqlId: {} unlock successful !", sqlId);
+            return true;
+        } else {
+            log.warn("sqlId: {} unlock. but sql is unlocked !", sqlId);
+            return false;
+        }
+    }
+
+    private static final String sqlTemp = "INSERT OVERWRITE TABLE winhc_ng.out_pull_spider_data PARTITION(ds) SELECT to_json(Map({maps})) AS detail_json ,CONCAT_ws('_','{random}',rn) AS ds FROM ( SELECT * ,ntile({partitionNum}) OVER({orderBy}) rn FROM ( {allTab} ) ) ;";
+
+    private void checkColumns(List<String> exportColumns, String tn) {
+        List<String> allColumns = odpsSchemaUtils.getAllColumns("winhc_ng", "inc_ads_" + tn);
+        allColumns.retainAll(exportColumns);
+        Assert.isTrue(allColumns.size() == exportColumns.size(), "要导出的列不存在 ！");
+    }
+
+    @Override
+    public OdpsSqlInfoVO registerSql(RegisterSqlVO registerSqlVO) {
+        String tn = registerSqlVO.getTn();
+        String json = registerSqlVO.getColumns().stream().map(r -> "\"" + r + "\"," + r).collect(Collectors.joining(","));
+        Integer partitionNum = registerSqlVO.getPartitionNum();
+        String registerBy = registerSqlVO.getRegisterBy();
+
+        checkColumns(registerSqlVO.getColumns(), tn);
+
+        String allSql = odpsSchemaUtils.getAllSql(tn, "winhc_ng", "ads", registerSqlVO.getWhere());
+
+        Map<String, String> map = new HashMap<String, String>(12) {
+            {
+                put("maps", json);
+                put("partitionNum", partitionNum + "");
+                put("allTab", allSql);
+            }
+        };
+
+        List<SortVO> sort = registerSqlVO.getSort();
+        String orderBy = "";
+        if (!sort.isEmpty()) {
+            orderBy = sort.stream().map(r -> {
+                if (r.getAse()) {
+                    return r.getField() + " ase";
+                } else {
+                    return r.getField() + " desc";
+                }
+            }).collect(Collectors.joining(",", "ORDER BY ", ""));
+        }
+        map.put("orderBy", orderBy);
+
+        String sql = StrUtil.format(sqlTemp, map);
+        OdpsSqlInfoVO odpsSqlInfoVO = OdpsSqlInfoVO.builder()
+                .sql(sql)
+                .outputTab("out_pull_spider_data")
+                .createBy(registerBy)
+                .partitions(partitionNum)
+                .build();
+        add(odpsSqlInfoVO);
+        return odpsSqlInfoVO;
+    }
+
+    private static final Pattern pattern = Pattern.compile("[\r\n ]+");
+
+    @Override
+    public String add(OdpsSqlInfoVO sqlInfo) {
+        String sql = pattern.matcher(sqlInfo.getSql()).replaceAll(" ").trim();
+        Assert.isTrue(StringUtils.isNotEmpty(sql), "sql为空！");
+        if (StringUtils.isEmpty(sqlInfo.getId())) {
+            sqlInfo.setId(DigestUtil.md5Hex(sql));
+        }
+        OdpsSqlInfo build = OdpsSqlInfo.builder()
+                .id(sqlInfo.getId())
+                .sqlStr(sql)
+                .createBy(sqlInfo.getCreateBy())
+                .outputTab(sqlInfo.getOutputTab())
+                .partitions(sqlInfo.getPartitions()).build();
+        saveOrUpdate(build);
+        return build.getId();
+    }
+
+    @Override
+    public OdpsSqlInfoVO get(String sqlId) {
+        OdpsSqlInfo byId = getById(sqlId);
+        return OdpsSqlInfoVO.builder()
+                .id(byId.getId())
+                .sql(byId.getSqlStr())
+                .outputTab(byId.getOutputTab())
+                .partitions(byId.getPartitions())
+                .createBy(byId.getCreateBy())
+                .build();
+
+    }
+
+    @Override
+    public Boolean check(String sqlId) {
+        return getById(sqlId) != null;
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/service/impl/PullRequestRecordServiceImpl.java

@@ -0,0 +1,47 @@
+package com.winhc.bigdata.task.framework.odps.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.winhc.bigdata.task.enums.TaskStatusEnum;
+import com.winhc.bigdata.task.framework.odps.entity.PullRequestRecord;
+import com.winhc.bigdata.task.framework.odps.mapper.PullRequestRecordMapper;
+import com.winhc.bigdata.task.framework.odps.service.OdpsSqlInfoService;
+import com.winhc.bigdata.task.framework.odps.service.PullRequestRecordService;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.springframework.util.Assert;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 10:09
+ */
+@Slf4j
+@Service
+@AllArgsConstructor
+public class PullRequestRecordServiceImpl extends ServiceImpl<PullRequestRecordMapper, PullRequestRecord> implements PullRequestRecordService {
+
+    private final OdpsSqlInfoService odpsSqlInfoService;
+
+
+    @Override
+    public synchronized Object pull(String sqlId, String pullBy) {
+        Assert.isTrue(odpsSqlInfoService.check(sqlId), "sql id 不存在！请先注册函数 ");
+        List<PullRequestRecord> list = lambdaQuery()
+                .eq(PullRequestRecord::getSqlId, sqlId)
+                .ne(PullRequestRecord::getPartitionTaskStatus, TaskStatusEnum.导出完毕)
+                .list();
+        if (list.size() >= 1) {
+            return "已经有任务在导出中... pull id:" + list.stream().map(PullRequestRecord::getId).map(r -> r + "").collect(Collectors.joining(","));
+        }
+        PullRequestRecord build = PullRequestRecord.builder()
+                .sqlId(sqlId)
+                .pullBy(pullBy)
+                .partitionTaskStatus(TaskStatusEnum.等待中)
+                .build();
+        save(build);
+        return true;
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/utils/ExecOdpsSqlUtil.java

@@ -0,0 +1,31 @@
+package com.winhc.bigdata.task.framework.odps.utils;
+
+import com.aliyun.odps.Instance;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.OdpsException;
+import com.aliyun.odps.task.SQLTask;
+import com.winhc.bigdata.task.framework.odps.vo.MaxComputeSqlInstance;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/1 15:57
+ */
+@Slf4j
+@Component
+@AllArgsConstructor
+public class ExecOdpsSqlUtil {
+
+    private final Odps odps;
+
+    public MaxComputeSqlInstance exec(String sql) throws OdpsException {
+        Instance run = SQLTask.run(odps, sql);
+        return MaxComputeSqlInstance.of(run);
+    }
+
+    public Instance getInstance(String instanceId) {
+        return odps.instances().get(instanceId);
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/utils/ExportFlowUtils.java

@@ -0,0 +1,40 @@
+package com.winhc.bigdata.task.framework.odps.utils;
+
+import com.winhc.bigdata.task.service.DataWorksService;
+import com.winhc.bigdata.task.service.DingTalkService;
+import com.winhc.bigdata.task.util.DateUtils;
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowTask;
+import lombok.AllArgsConstructor;
+import org.springframework.stereotype.Component;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 10:45
+ */
+@Component
+@AllArgsConstructor
+public class ExportFlowUtils {
+    private final DataWorksService dataWorksService;
+    private static final String project = "winhc_ng";
+    private static final String flow = "pull_spider_task";
+    private static final String task = "base";
+    private final Map<String, DataWorksFlowTask> map;
+    private final DingTalkService dingTalkService;
+    public Long export(String ds) {
+        String beanName = project + ":" + flow + ":" + task;
+        String yesterday = DateUtils.getYesterday();
+
+        DataWorksFlowTask dataWorksFlowTask = map.get(beanName);
+
+        Map<String, String> map = new HashMap<>();
+        map.put("ds", ds);
+        Long touch = dataWorksService.touch(project, flow, yesterday, dataWorksFlowTask, map);
+        dingTalkService.info("触发运维中心手动业务流程：{}  dag-id：{}; request body:{}", beanName, touch, map);
+        return touch ;
+
+    }
+
+}

src/main/java/com/winhc/bigdata/task/framework/odps/utils/OdpsSchemaUtils.java

@@ -0,0 +1,80 @@
+package com.winhc.bigdata.task.framework.odps.utils;
+
+import cn.hutool.core.util.StrUtil;
+import com.aliyun.odps.Column;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.Table;
+import com.aliyun.odps.TableSchema;
+import lombok.AllArgsConstructor;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/7 09:44
+ */
+@Component
+@AllArgsConstructor
+public class OdpsSchemaUtils {
+    private final Odps odps;
+
+    public List<String> getColumns(String projectName, String tableName) {
+        Table table = odps.tables().get(projectName, tableName);
+        TableSchema schema = table.getSchema();
+        List<Column> columns = schema.getColumns();
+        return columns.stream().map(Column::getName).collect(Collectors.toList());
+    }
+
+    public List<String> getPartitionColumns(String projectName, String tableName) {
+        Table table = odps.tables().get(projectName, tableName);
+        TableSchema schema = table.getSchema();
+        return schema.getPartitionColumns().stream().map(Column::getName).collect(Collectors.toList());
+    }
+
+    public List<String> getAllColumns(String projectName, String tableName) {
+        List<String> columns = getColumns(projectName, tableName);
+        columns.addAll(getPartitionColumns(projectName, tableName));
+        return columns;
+    }
+
+    private final String sqlTemp = "SELECT {cols} FROM ( SELECT * ,ROW_NUMBER() OVER(PARTITION BY {rowkey} ORDER BY ds DESC ) AS num FROM ( SELECT {cols} FROM {project}.{allTab} WHERE ds > 0 UNION ALL SELECT {cols} FROM {project}.{incTab} WHERE ds > 0 ) AS all_t1 ) AS all_t2 WHERE all_t2.num = 1 {where}";
+
+    public String getAllSql(String tn, String project, String prefix,String whereExpression) {
+        String allTab = prefix + "_" + tn;
+        String incTab = "inc_" + prefix + "_" + tn;
+
+        List<String> columns = getAllColumns(project, incTab);
+        columns.retainAll(getAllColumns(project, allTab));
+        String rowkey;
+        if (columns.contains("rowkey")) {
+            rowkey = "rowkey";
+        } else if (columns.contains("company_id")) {
+            rowkey = "company_id";
+        } else {
+            throw new RuntimeException("PARTITION BY 找不到对应的值 !");
+        }
+
+        String where = "";
+        if (StringUtils.isNotEmpty(whereExpression)) {
+            where = " AND (" + whereExpression + ")";
+        }
+        String finalWhere = where;
+        Map<String, String> map = new HashMap<String, String>() {
+            {
+                put("cols", String.join(",", columns));
+                put("allTab", allTab);
+                put("incTab", incTab);
+                put("project", project);
+                put("rowkey", rowkey);
+                put("where", finalWhere);
+            }
+        };
+        String format = StrUtil.format(sqlTemp, map);
+        return format;
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/vo/MaxComputeSqlInstance.java

@@ -0,0 +1,47 @@
+package com.winhc.bigdata.task.framework.odps.vo;
+
+import com.aliyun.odps.Instance;
+import com.aliyun.odps.OdpsException;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Date;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/2 10:19
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MaxComputeSqlInstance {
+    private String id;
+    private String jobName;
+    private Integer priority;
+    private String project;
+    private Date startTime;
+    private Date endTime;
+    private String status;
+    private String userAccount;
+    private Boolean successful;
+    private Boolean terminated;
+
+
+    public static MaxComputeSqlInstance of( Instance instance) throws OdpsException {
+        String id = instance.getId();
+        Date startTime = instance.getStartTime();
+        Date endTime = instance.getEndTime();
+        String jobName = instance.getJobName();
+        String project = instance.getProject();
+        String status = instance.getStatus().toString();
+        String owner = instance.getOwner();
+        boolean successful = instance.isSuccessful();
+        boolean terminated = instance.isTerminated();
+        int priority = instance.getPriority();
+        MaxComputeSqlInstance sqlInstance = new MaxComputeSqlInstance(id, jobName, priority, project, startTime, endTime, status, owner, successful, terminated);
+        return sqlInstance;
+    }
+}

src/main/java/com/winhc/bigdata/task/framework/odps/vo/OdpsSqlInfoVO.java

@@ -0,0 +1,25 @@
+package com.winhc.bigdata.task.framework.odps.vo;
+
+import lombok.Builder;
+import lombok.Data;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/5 16:35
+ */
+@Builder
+@Data
+public class OdpsSqlInfoVO {
+    private String id;
+
+    private String sql;
+
+    private String outputTab;
+
+    /**
+     * 分区数量
+     */
+    private Integer partitions;
+
+    private String createBy;
+}

src/main/java/com/winhc/bigdata/task/framework/odps/vo/RegisterSqlVO.java

@@ -0,0 +1,21 @@
+package com.winhc.bigdata.task.framework.odps.vo;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/7 10:12
+ */
+@Data
+@Builder
+public class RegisterSqlVO {
+    private String tn;
+    private List<String> columns;
+    private Integer partitionNum;
+    private String registerBy;
+    private List<SortVO> sort;
+    private String where;
+}

src/main/java/com/winhc/bigdata/task/framework/odps/vo/SortVO.java

@@ -0,0 +1,19 @@
+package com.winhc.bigdata.task.framework.odps.vo;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/7 10:23
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class SortVO {
+    private String field;
+    private Boolean ase;
+}

src/main/java/com/winhc/bigdata/task/jobs/RemoveCompanyIdLookupJob.java

@@ -0,0 +1,103 @@
+package com.winhc.bigdata.task.jobs;
+
+import com.mongodb.client.MongoCollection;
+import com.winhc.bigdata.task.service.DingTalkService;
+import com.winhc.bigdata.task.util.EsFastScan;
+import com.winhc.bigdata.task.util.MaxBatchQueryUtils;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.bson.Document;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.stereotype.Component;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+/**
+ * @author: XuJiakai
+ * 2021/6/18 16:18
+ */
+@Slf4j
+@Component
+@AllArgsConstructor
+public class RemoveCompanyIdLookupJob {
+    private final DingTalkService dingTalkService;
+    private final RestHighLevelClient restHighLevelClient;
+    private final MongoTemplate mongoTemplate;
+    private final Table2CompanyIdMapProperties table2CompanyIdMapProperties;
+
+
+    @KafkaListener(id = "RemoveCompanyIdLookupJob"
+            , topics = "inc_update_dimension"
+            , groupId = "${spring.kafka.consumer.group-id:xjk_group}", containerFactory = "containerFactory")
+    public void removeCompanyIdLookupJob(List<ConsumerRecord<String, String>> records) {
+        List<String> collect = records.stream().map(ConsumerRecord::value).collect(Collectors.toList());
+        exec(collect);
+    }
+
+
+    public Boolean exec(List<String> companyId) {
+        for (String key : table2CompanyIdMapProperties.getMap().keySet()) {
+            try {
+                exec(companyId, key);
+            } catch (Exception e) {
+                MongoCollection<Document> error = mongoTemplate.getCollection("RemoveCompanyIdLookupJob_error");
+                List<Document> rowkey = companyId.stream().map(r -> {
+                    Document document = new Document();
+                    document.put("company_id", r);
+                    document.put("tn", key);
+                    return document;
+                }).collect(Collectors.toList());
+                error.insertMany(rowkey);
+                log.error(e.getMessage(), e);
+                dingTalkService.error("RemoveCompanyIdLookupJob {}", e.getMessage());
+            }
+        }
+        return true;
+    }
+
+
+    public Boolean exec(List<String> companyIds, String tn) {
+        if (!table2CompanyIdMapProperties.getMap().containsKey(tn)) {
+            dingTalkService.error("RemoveCompanyIdLookupJob 移除公司关联id查找，存在未知tn: {}", tn);
+            return false;
+        }
+        if (companyIds.isEmpty()) {
+            return true;
+        }
+        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+        String s = table2CompanyIdMapProperties.getMap().get(tn);
+        if (s.contains(",")) {
+            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+            for (String k : s.split(",")) {
+                boolQueryBuilder.should(MaxBatchQueryUtils.addTerms(companyIds, k));
+            }
+            boolQuery.filter(boolQueryBuilder);
+        } else {
+            boolQuery.filter(MaxBatchQueryUtils.addTerms(companyIds, s));
+        }
+
+
+        MongoCollection<Document> tab = mongoTemplate.getCollection("company_back_update_0423");
+        Consumer<SearchHit[]> func = list -> {
+            List<Document> li = Arrays.stream(list).map(d -> {
+                String id = d.getId();
+                Document document = new Document();
+                document.put("rowkey", id);
+                document.put("tn", tn);
+                return document;
+            }).collect(Collectors.toList());
+            tab.insertMany(li);
+        };
+        new EsFastScan(restHighLevelClient, func, "winhc_index_" + tn, "_doc", boolQuery).scan();
+        return true;
+    }
+}

src/main/java/com/winhc/bigdata/task/jobs/Table2CompanyIdMapProperties.java

@@ -0,0 +1,20 @@
+package com.winhc.bigdata.task.jobs;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+
+/**
+ * @author: XuJiakai
+ * 2021/6/18 17:41
+ */
+@Configuration
+@ConfigurationProperties(prefix = "table-to-company", ignoreUnknownFields = false)
+@Data
+@Component
+public class Table2CompanyIdMapProperties {
+    private Map<String,String> map ;
+}

src/main/java/com/winhc/bigdata/task/mapper/MaxComputeDetailOutputMapper.java

@@ -0,0 +1,16 @@
+package com.winhc.bigdata.task.mapper;
+
+import com.winhc.bigdata.task.entity.MaxComputeDetailOutput;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * @Entity com.winhc.bigdata.task.entity.MaxComputeDetailOutput
+ */
+
+public interface MaxComputeDetailOutputMapper extends BaseMapper<MaxComputeDetailOutput> {
+
+}
+
+
+
+

src/main/java/com/winhc/bigdata/task/mapper/PullDataLogMapper.java

@@ -0,0 +1,15 @@
+package com.winhc.bigdata.task.mapper;
+
+import com.winhc.bigdata.task.entity.PullDataLog;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * @Entity com.winhc.bigdata.task.entity.PullDataLog
+ */
+public interface PullDataLogMapper extends BaseMapper<PullDataLog> {
+
+}
+
+
+
+

src/main/java/com/winhc/bigdata/task/mapper/PullDataTaskMapper.java

@@ -0,0 +1,16 @@
+package com.winhc.bigdata.task.mapper;
+
+import com.winhc.bigdata.task.entity.PullDataTask;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * @author x
+ * @Entity com.winhc.bigdata.task.entity.PullDataTask
+ */
+public interface PullDataTaskMapper extends BaseMapper<PullDataTask> {
+
+}
+
+
+
+

src/main/java/com/winhc/bigdata/task/service/CompanyIndexService.java

@@ -0,0 +1,11 @@
+package com.winhc.bigdata.task.service;
+
+import java.util.Map;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/13 14:28
+ */
+public interface CompanyIndexService {
+    Map<String, Long> getDataDistribution();
+}

src/main/java/com/winhc/bigdata/task/service/DataWorksService.java

@@ -0,0 +1,23 @@
+package com.winhc.bigdata.task.service;
+
+import com.winhc.bigdata.task.enums.DataWorksTaskFlowEnum;
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowTask;
+
+import java.util.Map;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/14 14:09
+ */
+public interface DataWorksService {
+
+    Long exportData(String dt);
+
+    Long generateData(String taskId);
+
+
+    Long touch(String project, String flowName, String bizDate, DataWorksFlowTask dataWorksFlowTask, Map<String, String> otherParams);
+
+    Map<String, DataWorksTaskFlowEnum> query(String projectName, Long dagId);
+
+}

src/main/java/com/winhc/bigdata/task/service/DingTalkService.java

@@ -0,0 +1,15 @@
+package com.winhc.bigdata.task.service;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/12 14:05
+ */
+public interface DingTalkService {
+    boolean send(String msg);
+
+    boolean info(String infoMsgTemplate, Object... params);
+
+    boolean warn(String warnMsgTemplate, Object... params);
+
+    boolean error(String errorMsgTemplate, Object... params);
+}

src/main/java/com/winhc/bigdata/task/service/MaxComputeDetailOutputService.java

@@ -0,0 +1,11 @@
+package com.winhc.bigdata.task.service;
+
+import com.winhc.bigdata.task.entity.MaxComputeDetailOutput;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ *
+ */
+public interface MaxComputeDetailOutputService extends IService<MaxComputeDetailOutput> {
+
+}

src/main/java/com/winhc/bigdata/task/service/PullDataLogService.java

@@ -0,0 +1,11 @@
+package com.winhc.bigdata.task.service;
+
+import com.winhc.bigdata.task.entity.PullDataLog;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ *
+ */
+public interface PullDataLogService extends IService<PullDataLog> {
+
+}

src/main/java/com/winhc/bigdata/task/service/PullDataService.java

@@ -0,0 +1,26 @@
+package com.winhc.bigdata.task.service;
+
+import com.winhc.bigdata.task.entity.MaxComputeDetailOutput;
+import com.winhc.bigdata.task.vo.TablePartitionInfoVO;
+
+import java.util.List;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/12 13:40
+ */
+public interface PullDataService {
+
+    /**
+     * @return
+     */
+    Object pull();
+
+    /**
+     * @param tab
+     * @return
+     */
+    Object callBack(List<TablePartitionInfoVO> tab);
+
+    void queryWait(MaxComputeDetailOutput detailOutput, Long pullId, Long dagId);
+}

src/main/java/com/winhc/bigdata/task/service/PullDataTaskService.java

@@ -0,0 +1,11 @@
+package com.winhc.bigdata.task.service;
+
+import com.winhc.bigdata.task.entity.PullDataTask;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * @author x
+ */
+public interface PullDataTaskService extends IService<PullDataTask> {
+
+}

src/main/java/com/winhc/bigdata/task/service/TaskCenterService.java

@@ -0,0 +1,15 @@
+package com.winhc.bigdata.task.service;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/1 15:43
+ */
+public interface TaskCenterService {
+
+    /**
+     * @return
+     */
+    Object pull();
+
+
+}

src/main/java/com/winhc/bigdata/task/service/impl/CompanyIndexServiceImpl.java

@@ -0,0 +1,58 @@
+package com.winhc.bigdata.task.service.impl;
+
+import com.winhc.bigdata.task.dao.CompanyIndexDao;
+import com.winhc.bigdata.task.service.CompanyIndexService;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedRange;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.springframework.stereotype.Service;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/13 14:28
+ */
+@Slf4j
+@Service
+@AllArgsConstructor
+public class CompanyIndexServiceImpl implements CompanyIndexService {
+    private final CompanyIndexDao companyIndexDao;
+
+
+    @Override
+    public Map<String, Long> getDataDistribution() {
+        RangeAggregationBuilder agg = AggregationBuilders.range("score_agg").field("company_score_weight");
+        agg.addRange(new RangeAggregator.Range("0", null, 1.0));
+        agg.addRange("1", 1, 2);
+        agg.addRange("2", 2, 3);
+        agg.addRange("3", 3, 4);
+        agg.addRange("4", 4, 5);
+        agg.addRange("5", 5, 6);
+        agg.addRange("6", 6, 7);
+        agg.addRange("7", 7, 8);
+        agg.addRange("8", 8, 9);
+        agg.addRange("9", 9, 10);
+        agg.addRange(new RangeAggregator.Range("10", 10.0, null));
+
+
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(matchAllQuery()).size(0)
+                .aggregation(agg);
+
+        SearchResponse search = companyIndexDao.search(sourceBuilder);
+        ParsedRange scoreAgg = search.getAggregations().get("score_agg");
+
+        return scoreAgg.getBuckets()
+                .stream()
+                .map(o -> (ParsedRange.ParsedBucket) o)
+                .collect(Collectors.toMap(ParsedRange.ParsedBucket::getKeyAsString, ParsedRange.ParsedBucket::getDocCount));
+    }
+}

src/main/java/com/winhc/bigdata/task/service/impl/DataWorksServiceImpl.java

@@ -0,0 +1,131 @@
+package com.winhc.bigdata.task.service.impl;
+
+import cn.hutool.core.codec.Base64;
+import com.aliyuncs.IAcsClient;
+import com.aliyuncs.dataworks_public.model.v20180601.CreateManualDagRequest;
+import com.aliyuncs.dataworks_public.model.v20180601.CreateManualDagResponse;
+import com.aliyuncs.dataworks_public.model.v20180601.SearchManualDagNodeInstanceRequest;
+import com.aliyuncs.dataworks_public.model.v20180601.SearchManualDagNodeInstanceResponse;
+import com.aliyuncs.http.ProtocolType;
+import com.winhc.bigdata.task.configuration.DataWorksAccessProperties;
+import com.winhc.bigdata.task.configuration.ProjectParamInit;
+import com.winhc.bigdata.task.enums.DataWorksTaskFlowEnum;
+import com.winhc.bigdata.task.service.CompanyIndexService;
+import com.winhc.bigdata.task.service.DataWorksService;
+import com.winhc.bigdata.task.util.DateUtils;
+import com.winhc.bigdata.task.util.JsonUtils;
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowTask;
+import com.winhc.bigdata.task.vo.dataworks.TaskParam;
+import lombok.AllArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/14 14:10
+ */
+@Slf4j
+@Service
+@AllArgsConstructor
+public class DataWorksServiceImpl implements DataWorksService {
+
+    private final DataWorksAccessProperties dataWorksAccessProperties;
+    private final IAcsClient client;
+    private final Map<String, DataWorksFlowTask> dataWorksFlowTaskMap;
+    private final CompanyIndexService companyIndexService;
+
+    @SneakyThrows
+    public CreateManualDagResponse touch(String projectName
+            , String flowName, String bizDate) {
+        return triggerWithParam(projectName, flowName, bizDate, null, null);
+    }
+
+    @SneakyThrows
+    public CreateManualDagResponse triggerWithParam(String projectName
+            , String flowName, String bizDate, String dagPara, String nodePara) {
+        log.info("触发任务：{}.{} {}\n{}\n{}", projectName, flowName, bizDate,dagPara,nodePara);
+        CreateManualDagRequest request = new CreateManualDagRequest();
+        request.setProjectName(projectName);
+        request.setFlowName(flowName);
+        request.setBizdate(bizDate + " 00:00:00");
+        request.setRegionId(dataWorksAccessProperties.getRegionId());
+        request.setProtocol(ProtocolType.HTTP);
+        if (StringUtils.isNotEmpty(dagPara)) {
+            request.setDagPara(dagPara);
+        }
+        if (StringUtils.isNotEmpty(nodePara)) {
+            request.setNodePara(nodePara);
+        }
+        CreateManualDagResponse response = client.getAcsResponse(request);
+        log.info("\n任务结果：\n\trequestId:{},\n\treturnCode:{},\n\treturnErrorSolution:{},\n\treturnMessage:{},\n\treturnValue:{}",
+                response.getRequestId()
+                , response.getReturnCode()
+                , response.getReturnErrorSolution()
+                , response.getReturnMessage()
+                , response.getReturnValue());
+        return response;
+    }
+
+    @SneakyThrows
+    public CreateManualDagResponse touch(TaskParam taskParam) {
+        return triggerWithParam(taskParam.getProjectName(), taskParam.getFlowName(), taskParam.getBizDate(), null, JsonUtils.jsonObjToString(taskParam.getNodeParam()));
+    }
+
+
+    @Override
+    public Long exportData(String dt) {
+        String yesterday = DateUtils.getYesterday();
+        HashMap<String, String> map = new HashMap<>();
+        map.put("dt", dt);
+        return touch(ProjectParamInit.project, ProjectParamInit.exportFlowName, yesterday, dataWorksFlowTaskMap.get(ProjectParamInit.exportBeanName), map);
+    }
+
+    @Override
+    public Long generateData(String taskId) {
+        Map<String, Long> dataDistribution = companyIndexService.getDataDistribution();
+        String s = JsonUtils.jsonObjToString(dataDistribution);
+        String yesterday = DateUtils.getYesterday();
+        HashMap<String, String> map = new HashMap<>();
+        map.put("taskId", taskId);
+        map.put("dataDistribution", Base64.encode(s));
+        return touch(ProjectParamInit.project, ProjectParamInit.generateFlowName, yesterday, dataWorksFlowTaskMap.get(ProjectParamInit.generateBeanName), map);
+    }
+
+    @Override
+    public Long touch(String project, String flowName, String bizDate, DataWorksFlowTask dataWorksFlowTask, Map<String, String> otherParams) {
+        TaskParam build = TaskParam.builder()
+                .projectName(project)
+                .bizDate(bizDate)
+                .flowName(flowName)
+                .nodeParam(dataWorksFlowTask.toNodeParam(bizDate, otherParams))
+                .build();
+        CreateManualDagResponse touch = touch(build);
+        return touch.getReturnValue();
+    }
+
+    @SneakyThrows
+    @Override
+    public Map<String, DataWorksTaskFlowEnum> query(String projectName, Long dagId) {
+        SearchManualDagNodeInstanceRequest searchNodeInstanceListRequest
+                = new SearchManualDagNodeInstanceRequest();
+        searchNodeInstanceListRequest.setDagId(dagId);
+        searchNodeInstanceListRequest.setProjectName(projectName);
+        searchNodeInstanceListRequest.setProtocol(ProtocolType.HTTP);
+        SearchManualDagNodeInstanceResponse searchResponse = client
+                .getAcsResponse(searchNodeInstanceListRequest);
+        java.util.List<SearchManualDagNodeInstanceResponse.NodeInsInfo> nodeInsfos = searchResponse.getData();
+        for (SearchManualDagNodeInstanceResponse.NodeInsInfo nodeInsInfo : nodeInsfos) {
+            DataWorksTaskFlowEnum code = DataWorksTaskFlowEnum.getTaskFlowEnumByCode(nodeInsInfo.getStatus());
+            log.info("{}:{} {}", nodeInsInfo.getNodeName(), nodeInsInfo.getStatus(), code);
+        }
+        return nodeInsfos.stream()
+                .collect(Collectors.toMap(SearchManualDagNodeInstanceResponse.NodeInsInfo::getNodeName, node -> DataWorksTaskFlowEnum.getTaskFlowEnumByCode(node.getStatus()), (o1, o2) -> o1));
+
+    }
+}

src/main/java/com/winhc/bigdata/task/service/impl/DataWorksServiceImplTest.java

@@ -0,0 +1,66 @@
+package com.winhc.bigdata.task.service.impl;
+
+import cn.hutool.core.codec.Base64;
+import com.winhc.bigdata.task.configuration.ProjectParamInit;
+import com.winhc.bigdata.task.enums.DataWorksTaskFlowEnum;
+import com.winhc.bigdata.task.service.CompanyIndexService;
+import com.winhc.bigdata.task.service.DataWorksService;
+import com.winhc.bigdata.task.util.JsonUtils;
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowTask;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/14 15:38
+ */
+@Slf4j
+//@Service
+@AllArgsConstructor
+public class DataWorksServiceImplTest implements DataWorksService {
+    private final CompanyIndexService companyIndexService;
+    private final Map<String, DataWorksFlowTask> dataWorksFlowTaskMap;
+
+    @Override
+    public Long exportData(String dt) {
+        log.info(dt);
+        return 123L;
+    }
+
+    @Override
+    public Long generateData(String taskId) {
+        Map<String, Long> dataDistribution = companyIndexService.getDataDistribution();
+        String s = JsonUtils.jsonObjToString(dataDistribution);
+        HashMap<String, String> map = new HashMap<>();
+        map.put("taskId", taskId);
+        map.put("dataDistribution", Base64.encode(s));
+        DataWorksFlowTask dataWorksFlowTask = dataWorksFlowTaskMap.get(ProjectParamInit.generateBeanName);
+        log.info("{}", dataWorksFlowTask.toNodeParam(map));
+        return 123L;
+    }
+
+    @Override
+    public Long touch(String project, String flowName, String bizDate, DataWorksFlowTask dataWorksFlowTask, Map<String, String> otherParams) {
+        Map<String, String> stringStringMap = dataWorksFlowTask.toNodeParam(bizDate, otherParams);
+        log.info(flowName);
+        log.info("{}", stringStringMap);
+        return 123L;
+    }
+
+    @Override
+    public Map<String, DataWorksTaskFlowEnum> query(String projectName, Long dagId) {
+        try {
+            Thread.sleep(3000);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        Map<String, DataWorksTaskFlowEnum> map = new HashMap<>();
+        DataWorksTaskFlowEnum d = DataWorksTaskFlowEnum.values()[new Random().nextInt(DataWorksTaskFlowEnum.values().length)];
+        map.put("abc", d);
+        return map;
+    }
+}

src/main/java/com/winhc/bigdata/task/service/impl/DingTalkServiceImpl.java

@@ -0,0 +1,57 @@
+package com.winhc.bigdata.task.service.impl;
+
+import cn.hutool.core.util.StrUtil;
+import com.winhc.bigdata.task.constant.DingTalkConst;
+import com.winhc.bigdata.task.service.DingTalkService;
+import com.winhc.bigdata.task.util.DingTalkUtils;
+import com.winhc.bigdata.task.util.OkHttpUtils;
+import com.winhc.bigdata.task.util.ding.talk.DingTalkText;
+import com.winhc.bigdata.task.vo.Entry;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/12 14:05
+ */
+@Slf4j
+@Service
+@AllArgsConstructor
+public class DingTalkServiceImpl implements DingTalkService {
+    private OkHttpUtils client;
+    private static final String URL = "https://oapi.dingtalk.com/robot/send?access_token=";
+
+    @Override
+    public boolean send(String text) {
+        return send(DingTalkText.builder().content(text).build().packString(), DingTalkConst.winhc_log_talk);
+    }
+
+    @Override
+    public boolean info(String infoMsgTemplate, Object... params) {
+        return send("【winhc-bigdata-task.info】" + StrUtil.format(infoMsgTemplate, params));
+    }
+
+    @Override
+    public boolean warn(String warnMsgTemplate, Object... params) {
+        return send("【winhc-bigdata-task.warn】" + StrUtil.format(warnMsgTemplate, params));
+    }
+
+    @Override
+    public boolean error(String errorMsgTemplate, Object... params) {
+        return send("【winhc-bigdata-task.error】" + StrUtil.format(errorMsgTemplate, params));
+    }
+
+    private boolean send(String body, DingTalkConst to) {
+        try {
+            Entry<Long, String> sign = DingTalkUtils.getSign(to.getDingSecret());
+            String query = to.getAccessToken() + "&timestamp=" + sign.getKey() + "&sign=" + sign.getValue();
+            String post = client.post(URL + query, body);
+            log.info(post);
+            return post.contains("ok");
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+            return false;
+        }
+    }
+}

src/main/java/com/winhc/bigdata/task/service/impl/MaxComputeDetailOutputServiceImpl.java

@@ -0,0 +1,20 @@
+package com.winhc.bigdata.task.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.winhc.bigdata.task.entity.MaxComputeDetailOutput;
+import com.winhc.bigdata.task.service.MaxComputeDetailOutputService;
+import com.winhc.bigdata.task.mapper.MaxComputeDetailOutputMapper;
+import org.springframework.stereotype.Service;
+
+/**
+ *
+ */
+@Service
+public class MaxComputeDetailOutputServiceImpl extends ServiceImpl<MaxComputeDetailOutputMapper, MaxComputeDetailOutput>
+        implements MaxComputeDetailOutputService {
+
+}
+
+
+
+

src/main/java/com/winhc/bigdata/task/service/impl/PullDataLogServiceImpl.java

@@ -0,0 +1,20 @@
+package com.winhc.bigdata.task.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.winhc.bigdata.task.entity.PullDataLog;
+import com.winhc.bigdata.task.service.PullDataLogService;
+import com.winhc.bigdata.task.mapper.PullDataLogMapper;
+import org.springframework.stereotype.Service;
+
+/**
+ *
+ */
+@Service
+public class PullDataLogServiceImpl extends ServiceImpl<PullDataLogMapper, PullDataLog>
+        implements PullDataLogService {
+
+}
+
+
+
+

src/main/java/com/winhc/bigdata/task/service/impl/PullDataServiceImpl.java

@@ -0,0 +1,323 @@
+package com.winhc.bigdata.task.service.impl;
+
+import cn.hutool.core.lang.Assert;
+import com.winhc.bigdata.task.configuration.ProjectParamInit;
+import com.winhc.bigdata.task.entity.MaxComputeDetailOutput;
+import com.winhc.bigdata.task.entity.PullDataLog;
+import com.winhc.bigdata.task.entity.PullDataTask;
+import com.winhc.bigdata.task.enums.DataWorksTaskFlowEnum;
+import com.winhc.bigdata.task.enums.TaskStatusEnum;
+import com.winhc.bigdata.task.service.*;
+import com.winhc.bigdata.task.util.GenerateIdUtils;
+import com.winhc.bigdata.task.util.JsonUtils;
+import com.winhc.bigdata.task.vo.ResponseVo;
+import com.winhc.bigdata.task.vo.TablePartitionInfoVO;
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowTask;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.ApplicationContext;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/12 13:40
+ */
+@Slf4j
+@Service
+@AllArgsConstructor
+public class PullDataServiceImpl implements PullDataService {
+    private final ApplicationContext applicationContext;
+
+    private final MaxComputeDetailOutputService maxComputeDetailOutputService;
+    private final PullDataTaskService pullDataTaskService;
+    private final PullDataLogService pullDataLogService;
+    private final DataWorksService dataWorksService;
+    private final Map<String, DataWorksFlowTask> dataWorksFlowTaskMap;
+    private final DingTalkService dingTalkService;
+
+
+    @PostConstruct
+    public void init() {
+        PullDataLog pullDataLog = pullDataLogService.lambdaQuery().eq(PullDataLog::getPullStatus, TaskStatusEnum.导出中).one();
+        MaxComputeDetailOutput maxComputeDetailOutput = maxComputeDetailOutputService.lambdaQuery().eq(MaxComputeDetailOutput::getPartitionTaskStatus, TaskStatusEnum.生成中).one();
+        if (pullDataLog != null && maxComputeDetailOutput != null) {
+            log.info("存在运行中的任务...");
+            applicationContext.getBean(PullDataService.class).queryWait(maxComputeDetailOutput, pullDataLog.getId(), maxComputeDetailOutput.getDagId());
+        }
+    }
+
+
+    @Override
+    public synchronized ResponseVo pull() {
+        Integer count = pullDataLogService.lambdaQuery()
+                .eq(PullDataLog::getPullStatus, TaskStatusEnum.导出中)
+                .count();
+        if (count != 0) {
+            return ResponseVo.success(getExportDataStatus(), "已有数据在导出中,请稍后...");
+        }
+
+        PullDataLog we = pullDataLogService.lambdaQuery()
+                .eq(PullDataLog::getPullStatus, TaskStatusEnum.等待中)
+                .one();
+
+        if (we != null) {
+            return export(we.getId());
+        } else {
+            PullDataLog pullDataLog = PullDataLog.builder()
+                    .pullBy("xjk")
+                    .pullStatus(TaskStatusEnum.等待中)
+                    .requestTime(new Date())
+                    .build();
+            pullDataLogService.save(pullDataLog);
+            return export(pullDataLog.getId());
+        }
+    }
+
+    @Override
+    public synchronized Object callBack(List<TablePartitionInfoVO> tab) {
+        Assert.isFalse(tab.isEmpty(), "maxcompute回调数据为空！");
+
+        String taskId = tab.get(0).getTaskId();
+        PullDataTask byId = pullDataTaskService.getById(taskId);
+
+        Assert.notNull(byId, "回调的taskId: {} 不存在", taskId);
+
+        Assert.isFalse(TaskStatusEnum.导出完毕 == byId.getTaskStatus() || TaskStatusEnum.生成完毕 == byId.getTaskStatus(), "请勿重复回调");
+
+
+        pullDataTaskService.lambdaUpdate()
+                .eq(PullDataTask::getId, taskId)
+                .set(PullDataTask::getEndTime, new Date())
+                .set(PullDataTask::getTaskStatus, TaskStatusEnum.生成完毕)
+                .update();
+
+        List<MaxComputeDetailOutput> collect = tab.stream().map(e ->
+                MaxComputeDetailOutput.builder()
+                        .taskId(e.getTaskId())
+                        .maxComputeTableName(e.getMaxComputeTableName())
+                        .partitionName(e.getPartitionName())
+                        .recordCount(e.getRecordCount())
+                        .priorityLevel(e.getPriorityLevel())
+                        .partitionTaskStatus(TaskStatusEnum.生成完毕).build()
+        ).collect(Collectors.toList());
+        maxComputeDetailOutputService.saveBatch(collect);
+
+        List<String> ps = tab.stream().map(TablePartitionInfoVO::getPartitionName).collect(Collectors.toList());
+
+        dingTalkService.info("数据生成完毕，共计生成{}个分区，分区列表：{}", tab.size(), JsonUtils.jsonObjToString(ps));
+
+        PullDataLog we = pullDataLogService.lambdaQuery()
+                .eq(PullDataLog::getPullStatus, TaskStatusEnum.等待中)
+                .one();
+        if (we != null) {
+            return export(we.getId());
+        } else {
+            return null;
+        }
+    }
+
+
+    private ResponseVo export(Long pullId) {
+        PullDataTask pullDataTask = pullDataTaskService
+                .lambdaQuery()
+                .eq(PullDataTask::getTaskStatus, TaskStatusEnum.生成完毕)
+                .one();
+
+        if (pullDataTask == null) {
+            PullDataTask one = pullDataTaskService
+                    .lambdaQuery()
+                    .eq(PullDataTask::getTaskStatus, TaskStatusEnum.生成中)
+                    .one();
+
+            if (one == null) {
+                //重新生成数据
+                PullDataTask pullDataTask1 = generatingData();
+                return ResponseVo.success(pullDataTask1, "数据已经提交生成...");
+            } else {
+                //已有生成数据任务在队列中
+                return ResponseVo.success(getGeneratingDataStatus(), "数据已经在生成中...");
+            }
+        } else {
+            //直接导出
+            String taskId = pullDataTask.getId();
+
+            MaxComputeDetailOutput detailOutput = maxComputeDetailOutputService
+                    .lambdaQuery()
+                    .eq(MaxComputeDetailOutput::getTaskId, taskId)
+                    .eq(MaxComputeDetailOutput::getPartitionTaskStatus, TaskStatusEnum.生成完毕)
+                    .orderByDesc(MaxComputeDetailOutput::getPriorityLevel)
+                    .list()
+                    .get(0);
+            export(detailOutput, pullId);
+            return ResponseVo.success(getExportDataStatus(), "数据正在导出中....");
+        }
+    }
+
+    /**
+     * 提交导出任务
+     * 提交线程池
+     *
+     * @param detailOutput
+     * @return
+     */
+    public void export(MaxComputeDetailOutput detailOutput, Long pullId) {
+        Long dagId = dataWorksService.exportData(detailOutput.getPartitionName());
+        Assert.notNull(dagId, "触发业务流程失败：{}", detailOutput);
+
+        pullDataLogService.lambdaUpdate()
+                .eq(PullDataLog::getId, pullId)
+                .set(PullDataLog::getPullStatus, TaskStatusEnum.导出中)
+                .update();
+
+        maxComputeDetailOutputService.lambdaUpdate()
+                .eq(MaxComputeDetailOutput::getId, detailOutput.getId())
+                .set(MaxComputeDetailOutput::getPartitionTaskStatus, TaskStatusEnum.导出中)
+                .set(MaxComputeDetailOutput::getDagId, dagId)
+                .update();
+
+        log.info("导出数据任务提交成功， {}", detailOutput);
+
+        applicationContext.getBean(PullDataService.class).queryWait(detailOutput, pullId, dagId);
+    }
+
+    @Override
+    @Async("export-data-pool")
+    public void queryWait(MaxComputeDetailOutput detailOutput, Long pullId, Long dagId) {
+        while (true) {
+            log.info("{} 运行中...", detailOutput);
+            Map<String, DataWorksTaskFlowEnum> query = dataWorksService.query(ProjectParamInit.project, dagId);
+            long count = query.values()
+                    .stream()
+                    .filter(e -> !(DataWorksTaskFlowEnum.SUCCESS.equals(e) || DataWorksTaskFlowEnum.FAILURE.equals(e)))
+                    .count();
+
+            if (count != 0) {
+                try {
+                    Thread.sleep(100000);
+                } catch (InterruptedException e) {
+                    log.error(e.getMessage(), e);
+                }
+            } else {
+                log.info("end");
+                break;
+            }
+        }
+
+        Map<String, DataWorksTaskFlowEnum> query = dataWorksService.query(ProjectParamInit.project, dagId);
+        long count = query.values()
+                .stream()
+                .filter(DataWorksTaskFlowEnum.FAILURE::equals)
+                .count();
+        if (count == 0) {
+            exportDataCallBack(detailOutput, pullId);
+        } else {
+            //失败数据回滚
+            pullDataLogService.lambdaUpdate()
+                    .eq(PullDataLog::getId, pullId)
+                    .set(PullDataLog::getPullStatus, TaskStatusEnum.等待中)
+                    .update();
+
+            maxComputeDetailOutputService.lambdaUpdate()
+                    .eq(MaxComputeDetailOutput::getId, detailOutput.getId())
+                    .set(MaxComputeDetailOutput::getPartitionTaskStatus, TaskStatusEnum.生成完毕)
+                    .update();
+
+            log.error("存在任务失败！dagId: {}", dagId);
+            dingTalkService.error("导出任务失败，dagId：{}", dagId);
+        }
+    }
+
+    /**
+     * 导出数据回调
+     * 更新状态
+     *
+     * @param detailOutput
+     * @param pullId
+     * @return
+     */
+    private boolean exportDataCallBack(MaxComputeDetailOutput detailOutput, Long pullId) {
+        pullDataLogService.lambdaUpdate()
+                .eq(PullDataLog::getId, pullId)
+                .set(PullDataLog::getPullStatus, TaskStatusEnum.导出完毕)
+                .update();
+
+        maxComputeDetailOutputService.lambdaUpdate()
+                .eq(MaxComputeDetailOutput::getId, detailOutput.getId())
+                .set(MaxComputeDetailOutput::getPartitionTaskStatus, TaskStatusEnum.导出完毕)
+                .update();
+
+        Integer count = maxComputeDetailOutputService.lambdaQuery()
+                .eq(MaxComputeDetailOutput::getTaskId, detailOutput.getTaskId())
+                .eq(MaxComputeDetailOutput::getPartitionTaskStatus, TaskStatusEnum.生成完毕)
+                .count();
+        if (count == 0) {
+            pullDataTaskService.lambdaUpdate()
+                    .eq(PullDataTask::getId, detailOutput.getTaskId())
+                    .set(PullDataTask::getTaskStatus, TaskStatusEnum.导出完毕)
+                    .update();
+        }
+        dingTalkService.info("导出任务成功，导出分区号：{}  本批次数量：{}", detailOutput.getPartitionName(), detailOutput.getRecordCount());
+        return true;
+    }
+
+    /**
+     * 生成数据
+     *
+     * @return
+     */
+    private PullDataTask generatingData() {
+        String taskId = GenerateIdUtils.generateTaskId();
+
+        Long dagId = dataWorksService.generateData(taskId);
+
+        PullDataTask build = PullDataTask.builder().id(taskId)
+                .startTime(new Date())
+                .dagId(dagId)
+                .maxComputeFlowName(ProjectParamInit.generateFlowName)
+                .taskStatus(TaskStatusEnum.生成中)
+                .build();
+
+        pullDataTaskService.save(build);
+        return build;
+    }
+
+
+    /**
+     * 生成数据进度
+     *
+     * @return
+     */
+    private Map<String, DataWorksTaskFlowEnum> getGeneratingDataStatus() {
+        PullDataTask one = pullDataTaskService
+                .lambdaQuery()
+                .eq(PullDataTask::getTaskStatus, TaskStatusEnum.生成中)
+                .one();
+        Long dagId = one.getDagId();
+        Assert.notNull(dagId, "dagId为空, {}", one);
+
+        return dataWorksService.query("winhc_ng", dagId);
+    }
+
+    /**
+     * 导出数据进度
+     *
+     * @return
+     */
+    private Map<String, DataWorksTaskFlowEnum> getExportDataStatus() {
+        MaxComputeDetailOutput one = maxComputeDetailOutputService
+                .lambdaQuery()
+                .eq(MaxComputeDetailOutput::getPartitionTaskStatus, TaskStatusEnum.导出中)
+                .one();
+        Long dagId = one.getDagId();
+        Assert.notNull(dagId, "dagId为空, {}", one);
+
+        return dataWorksService.query("winhc_ng", dagId);
+    }
+}

src/main/java/com/winhc/bigdata/task/service/impl/PullDataTaskServiceImpl.java

@@ -0,0 +1,20 @@
+package com.winhc.bigdata.task.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.winhc.bigdata.task.entity.PullDataTask;
+import com.winhc.bigdata.task.service.PullDataTaskService;
+import com.winhc.bigdata.task.mapper.PullDataTaskMapper;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author x
+ */
+@Service
+public class PullDataTaskServiceImpl extends ServiceImpl<PullDataTaskMapper, PullDataTask>
+        implements PullDataTaskService {
+
+}
+
+
+
+

src/main/java/com/winhc/bigdata/task/test/ScanEs.java

@@ -0,0 +1,38 @@
+package com.winhc.bigdata.task.test;
+
+import com.winhc.bigdata.task.util.EsFastScan;
+import lombok.AllArgsConstructor;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.search.SearchHit;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.Arrays;
+import java.util.function.Consumer;
+
+/**
+ * @author: XuJiakai
+ * 2021/6/18 19:52
+ */
+@Component
+@AllArgsConstructor
+public class ScanEs {
+    private final RestHighLevelClient restHighLevelClient;
+    private final KafkaTemplate<String, String> kafkaTemplate;
+
+    public void ex() {
+        String dsl = "{\"query\":{\"term\":{\"deleted\":{\"value\":\"9\"}}}}";
+
+        Consumer<SearchHit[]> func = list -> {
+            Arrays.stream(list).forEach(d -> {
+                String id = d.getId();
+                kafkaTemplate.send("", id);
+            });
+
+        };
+        new EsFastScan(restHighLevelClient, func, "winhc-company-v8", "company", dsl).scan();
+
+    }
+
+
+}

src/main/java/com/winhc/bigdata/task/util/BigDecimalUtil.java

@@ -0,0 +1,113 @@
+package com.winhc.bigdata.task.util;
+
+import java.math.BigDecimal;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/9/2 17:57
+ * @Description:
+ */
+public class BigDecimalUtil {
+    // 进行加法运算
+    public static double add(String v1, String v2) {
+
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.add(b2).doubleValue();
+    }
+
+    public static double add(double v1, double v2) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.add(b2).doubleValue();
+    }
+
+    // 进行减法运算
+    public static double subtract(String v1, String v2) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.subtract(b2).doubleValue();
+    }
+
+    public static double subtract(double v1, double v2) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.subtract(b2).doubleValue();
+    }
+
+    // 进行乘法运算
+    public static double mul(String d1, String d2) {
+        BigDecimal b1 = new BigDecimal(d1);
+        BigDecimal b2 = new BigDecimal(d2);
+        return b1.multiply(b2).doubleValue();
+    }
+
+    public static double mul(double d1, double d2) {
+        BigDecimal b1 = new BigDecimal(d1);
+        BigDecimal b2 = new BigDecimal(d2);
+        return b1.multiply(b2).doubleValue();
+    }
+
+    // 进行除法运算
+    public static double div(String d1, String d2, int len) {
+        BigDecimal b1 = new BigDecimal(d1);
+        BigDecimal b2 = new BigDecimal(d2);
+        if (b2.doubleValue() == 0) {
+            return 0;
+        }
+        return b1.divide(b2, len, BigDecimal.ROUND_HALF_UP).doubleValue();
+    }
+
+    public static double div(double d1, double d2, int len) {
+        if (d2 == 0) {
+            return 0;
+        }
+        BigDecimal b1 = new BigDecimal(d1);
+        BigDecimal b2 = new BigDecimal(d2);
+        return b1.divide(b2, len, BigDecimal.ROUND_HALF_UP).doubleValue();
+    }
+
+    public static double div(long d1, long d2, int len) {
+        if (d2 == 0) {
+            return 0;
+        }
+        BigDecimal b1 = new BigDecimal(d1);
+        BigDecimal b2 = new BigDecimal(d2);
+        return b1.divide(b2, len, BigDecimal.ROUND_HALF_UP).doubleValue();
+    }
+
+    // 进行四舍五入操作-len 小数点后保留几位
+    public static double round(double d, int len) {
+        BigDecimal b1 = new BigDecimal(d);
+        BigDecimal b2 = new BigDecimal(1);
+        // 任何一个数字除以1都是原数字
+        // ROUND_HALF_UP是BigDecimal的一个常量，表示进行四舍五入的操作
+        return b1.divide(b2, len, BigDecimal.ROUND_HALF_UP).doubleValue();
+    }
+
+
+    /**
+     * 进行四舍五入操作 -并保留指定位数
+     *
+     * @param @param  d1		处理数据
+     * @param @param  d2		如果为1 则自己保留多少位，100为除以100之后的结果再保留位数
+     * @param @param  len  小数点保留位数
+     * @param @return 参数
+     * @return String    返回类型
+     * @throws
+     * @Title: round
+     * @Description:
+     */
+    public static double round(String d1, String d2, int len) {
+        BigDecimal b1 = new BigDecimal(d1);
+        BigDecimal b2 = new BigDecimal(d2);
+        // 任何一个数字除以1都是原数字
+        // ROUND_HALF_UP是BigDecimal的一个常量，表示进行四舍五入的操作
+        double res = b1.divide(b2, len, BigDecimal.ROUND_HALF_UP).doubleValue();
+        return res;
+    }
+
+    public static Long double2Long(Double d) {
+        return Math.round(d);
+    }
+}

src/main/java/com/winhc/bigdata/task/util/DataWorksFlowStatusUtil.java

@@ -0,0 +1,32 @@
+package com.winhc.bigdata.task.util;
+
+import com.winhc.bigdata.task.enums.DataWorksTaskFlowEnum;
+
+import java.util.Map;
+
+/**
+ * @author: XuJiakai
+ * 2021/7/6 16:32
+ */
+public class DataWorksFlowStatusUtil {
+
+    public static Boolean isSuccessful(Map<String, DataWorksTaskFlowEnum> map) {
+        long count = map.values()
+                .stream()
+                .filter(e -> !(DataWorksTaskFlowEnum.SUCCESS.equals(e) || DataWorksTaskFlowEnum.FAILURE.equals(e)))
+                .count();
+        if (count > 0) {
+            return null;
+        }
+        long failureNum = map.values()
+                .stream()
+                .filter(e -> DataWorksTaskFlowEnum.FAILURE.equals(e))
+                .count();
+
+        if (failureNum > 0) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+}

src/main/java/com/winhc/bigdata/task/util/DataWorksParamUtils.java

@@ -0,0 +1,108 @@
+package com.winhc.bigdata.task.util;
+
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowJob;
+import com.winhc.bigdata.task.vo.dataworks.DataWorksFlowTask;
+import com.winhc.bigdata.task.vo.dataworks.NodeParam;
+import lombok.AllArgsConstructor;
+import lombok.SneakyThrows;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+import org.springframework.stereotype.Component;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.*;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/6/23 14:23
+ * @Description:
+ */
+@SuppressWarnings("all")
+@Component
+@AllArgsConstructor
+public class DataWorksParamUtils {
+//    @Autowired
+//    ResourceLoader resourceLoader;
+
+    private static final String defaultFileName = "data-works-param.yaml";
+
+    @SneakyThrows
+    public List<DataWorksFlowJob> getJobs() {
+        InputStream is = null;
+        InputStreamReader isr = null;
+        BufferedReader br = null;
+        try {
+//            File file =ResourceUtils.getFile("classpath:data-works-param.yaml");
+            Resource resource = new ClassPathResource("data-works-param.yaml");
+            is = resource.getInputStream();
+
+//            Resource resource = resourceLoader.getResource("classpath:data-works-param.yaml");
+//            is = resource.getInputStream();
+            isr = new InputStreamReader(is);
+            br = new BufferedReader(isr);
+            return parseJobs(br);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        } finally {
+            if (br != null) {
+                br.close();
+            }
+            if (isr != null) {
+                isr.close();
+            }
+            if (is != null) {
+                is.close();
+            }
+        }
+    }
+
+    public static List<DataWorksFlowJob> getJobs(String fileName) throws FileNotFoundException {
+        return parseJobs(fileName);
+    }
+
+    private static List<DataWorksFlowJob> parseJobs(String path) throws FileNotFoundException {
+        Reader reader = new FileReader(new File(path));
+        return parseJobs(reader);
+    }
+
+    private static List<DataWorksFlowJob> parseJobs(Reader reader) throws FileNotFoundException {
+        Yaml yml = new Yaml();
+        Map map = yml.loadAs(reader, Map.class);
+
+        List j = ((List) map.get("job"));
+        List<DataWorksFlowJob> jobs = (List<DataWorksFlowJob>) j
+                .stream()
+
+                .map(m -> {
+                    String project = ((String) ((Map<String, Object>) m).get("project"));
+                    String flow = ((String) ((Map<String, Object>) m).get("flow"));
+                    List<DataWorksFlowTask> collect1 = (List<DataWorksFlowTask>) ((List) ((Map<String, Object>) m).get("task"))
+                            .stream()
+                            .map(t -> {
+                                String taskName = ((String) ((Map<String, Object>) t).get("taskName"));
+                                List<Map<String, Object>> ll = ((List<Map<String, Object>>) ((Map<String, Object>) t).get("param"));
+                                List<NodeParam> nodeParamList = ll.stream()
+                                        .map(mm -> {
+                                            String nodeId = String.valueOf(mm.remove("_nodeId"));
+                                            Map<String, String> collect = mm.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> String.valueOf(e.getValue()), (o1, o2) -> o1));
+                                            return NodeParam.builder().nodeId(nodeId).param(collect).build();
+                                        }).collect(Collectors.toList());
+                                return new DataWorksFlowTask(taskName, nodeParamList);
+                            })
+                            .collect(Collectors.toList());
+                    DataWorksFlowJob build = DataWorksFlowJob.builder()
+                            .project(project)
+                            .flow(flow)
+                            .task(collect1)
+                            .build();
+                    return build;
+                })
+                .collect(Collectors.toList());
+
+        List<DataWorksFlowJob> collect = jobs.stream().collect(Collectors.groupingBy(m -> m.getFlow() + m.getProject())).values().stream().map(list -> new DataWorksFlowJob(list.get(0).getProject(), list.get(0).getFlow(), list.stream().flatMap(l -> l.getTask().stream()).collect(Collectors.toList()))).collect(Collectors.toList());
+        return collect;
+    }
+}

src/main/java/com/winhc/bigdata/task/util/DateUtils.java

@@ -0,0 +1,36 @@
+package com.winhc.bigdata.task.util;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.Locale;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/7/20 13:48
+ * @Description:
+ */
+public class DateUtils {
+    private static final DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd")
+            .withLocale(Locale.CHINA)
+            .withZone(ZoneId.systemDefault());
+    private static final DateTimeFormatter ymd_df = DateTimeFormatter.ofPattern("yyyyMMdd")
+            .withLocale(Locale.CHINA)
+            .withZone(ZoneId.systemDefault());
+
+    public static String getYesterday() {
+        Instant instant = Instant.now().minus(1, ChronoUnit.DAYS);
+        return df.format(instant);
+    }
+
+    public static String getYesterday_ymd() {
+        Instant instant = Instant.now().minus(1, ChronoUnit.DAYS);
+        return ymd_df.format(instant);
+    }
+
+    public static String getMinusDay(Integer i) {
+        Instant instant = Instant.now().minus(i, ChronoUnit.DAYS);
+        return df.format(instant);
+    }
+}

src/main/java/com/winhc/bigdata/task/util/DingTalkBody.java

@@ -0,0 +1,39 @@
+package com.winhc.bigdata.task.util;
+
+import com.winhc.bigdata.task.util.ding.talk.DingTalkMarkDown;
+import com.winhc.bigdata.task.util.ding.talk.DingTalkText;
+import com.winhc.bigdata.task.util.ding.talk.MarkDownBuilder;
+import com.winhc.bigdata.task.util.ding.talk.MarkDownText;
+import lombok.Builder;
+import lombok.Data;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/12 14:19
+ */
+@Builder
+@Data
+public class DingTalkBody {
+
+    public DingTalkText text(String text) {
+        return DingTalkText.builder().content(text).build();
+    }
+
+    public DingTalkMarkDown.DingTalkMarkDownBuilder markDown() {
+        return DingTalkMarkDown.builder();
+    }
+
+
+    public static void main(String[] args) {
+        MarkDownBuilder markDownBuilder = new MarkDownBuilder();
+        MarkDownBuilder 一级标题 = markDownBuilder.titleFor1("一级标题")
+                .titleFor2(new MarkDownText("fasdlkfj", MarkDownText.MarkDownTextStyle.删除线).link("https://www.baidu.com"))
+                .titleFor3("3")
+                .titleFor4("4")
+                .titleFor6("6");
+
+        System.out.println(一级标题);
+    }
+
+
+}

src/main/java/com/winhc/bigdata/task/util/DingTalkUtils.java

@@ -0,0 +1,32 @@
+package com.winhc.bigdata.task.util;
+
+import cn.hutool.crypto.digest.HMac;
+import cn.hutool.crypto.digest.HmacAlgorithm;
+import com.winhc.bigdata.task.vo.Entry;
+import org.apache.tomcat.util.codec.binary.Base64;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/12 14:15
+ */
+public class DingTalkUtils {
+
+    public static Entry<Long, String> getSign(String dingSecret) throws UnsupportedEncodingException {
+        Long timestamp = System.currentTimeMillis();
+        String stringToSign = timestamp + "\n" + dingSecret;
+
+
+        byte[] key = dingSecret.getBytes("UTF-8");
+        HMac mac = new HMac(HmacAlgorithm.HmacSHA256, key);
+        byte[] signData = mac.digest(stringToSign);
+
+        String sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), "UTF-8");
+        Entry<Long, String> entry = new Entry<>();
+        entry.setKey(timestamp);
+        entry.setValue(sign);
+        return entry;
+    }
+}

src/main/java/com/winhc/bigdata/task/util/EsFastScan.java

@@ -0,0 +1,149 @@
+package com.winhc.bigdata.task.util;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.elasticsearch.action.search.*;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
+import org.elasticsearch.search.Scroll;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+/**
+ * @author: XuJiakai
+ * 2020/11/16 19:23
+ */
+@Slf4j
+public class EsFastScan {
+    private final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(3L));
+
+    private RestHighLevelClient client;
+    private Consumer<SearchHit[]> func;
+    private String index;
+    private String type;
+    private QueryBuilder queryBuilder;
+    private int batchSize = 500;
+    private int thread_num = 10;
+
+    public EsFastScan(RestHighLevelClient client, Consumer<SearchHit[]> func, String index, String type) {
+        this.client = client;
+        this.func = func;
+        this.index = index;
+        this.type = type;
+    }
+
+    public EsFastScan(RestHighLevelClient client, Consumer<SearchHit[]> func, String index, String type, QueryBuilder queryBuilder) {
+        this.client = client;
+        this.func = func;
+        this.index = index;
+        this.type = type;
+        this.queryBuilder = queryBuilder;
+    }
+
+    public EsFastScan(RestHighLevelClient client, Consumer<SearchHit[]> func, String index, String type, String dsl) {
+        this.client = client;
+        this.func = func;
+        this.index = index;
+        this.type = type;
+        this.queryBuilder = getQuery(JSONObject.parseObject(dsl));
+    }
+
+
+    public EsFastScan batchSize(int batchSize) {
+        this.batchSize = batchSize;
+        return this;
+    }
+
+
+    @SneakyThrows
+    public void scan() {
+        int poolSize = thread_num;
+        ArrayBlockingQueue<Runnable> objects = new ArrayBlockingQueue<>(poolSize * 4);
+        ThreadPoolExecutor executorService = new ThreadPoolExecutor(
+                poolSize, poolSize * 2,
+                10L, TimeUnit.SECONDS,
+                objects,
+                new ThreadFactoryBuilder().setNameFormat("ScanEs-pool").build(),
+                new ThreadPoolExecutor.CallerRunsPolicy()
+        );
+
+
+        SearchRequest searchRequest = new SearchRequest(index).types(type).scroll(scroll);
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(batchSize);
+        if (queryBuilder != null) {
+            searchSourceBuilder.query(queryBuilder);
+        }
+
+        searchRequest.source(searchSourceBuilder);
+        SearchResponse searchResponse = client.search(searchRequest);
+
+        long totalHits = searchResponse.getHits().getTotalHits();
+        int n = searchResponse.getHits().getHits().length;
+        String scrollId = searchResponse.getScrollId();
+        if (n != 0) {
+            executorService.submit(() -> {
+                log.info("执行一批：{}", n);
+                func.accept(searchResponse.getHits().getHits());
+            });
+        } else {
+            log.warn("数据为空！");
+        }
+
+        do {
+            int num = scanData(executorService, scrollId);
+            if (num == 0) {
+                break;
+            }
+        } while (true);
+        log.info("es scan is successful，total hits: {}", totalHits);
+        try {
+            log.info("scroll id: {}", scrollId);
+            ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+            clearScrollRequest.addScrollId(scrollId);
+            client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
+            log.info("clean scroll id success !");
+        } catch (Exception e) {
+//            log.error(e.getMessage(), e);
+        }
+    }
+
+    @SneakyThrows
+    private int scanData(ThreadPoolExecutor executorService, String scrollId) {
+        SearchScrollRequest scroll = new SearchScrollRequest(scrollId);
+        SearchResponse searchResponse = client.scroll(scroll, RequestOptions.DEFAULT);
+        int num = searchResponse.getHits().getHits().length;
+        if (num == 0) {
+            return num;
+        }
+        executorService.submit(() -> {
+            log.info("执行一批：{}", num);
+            func.accept(searchResponse.getHits().getHits());
+        });
+        return num;
+    }
+
+
+    private static final String q = "query";
+
+    private static WrapperQueryBuilder getQuery(JSONObject jsonObject) {
+        String dsl_json;
+        if (jsonObject.containsKey(q)) {
+            dsl_json = JsonUtils.jsonObjToString(jsonObject.getJSONObject(q));
+        } else {
+            dsl_json = JsonUtils.jsonObjToString(jsonObject);
+        }
+        return QueryBuilders.wrapperQuery(dsl_json);
+    }
+}

src/main/java/com/winhc/bigdata/task/util/GenerateIdUtils.java

@@ -0,0 +1,31 @@
+package com.winhc.bigdata.task.util;
+
+import java.util.Random;
+
+/**
+ * @author: XuJiakai
+ * 2021/4/14 11:56
+ */
+public class GenerateIdUtils {
+    /**
+     * 获取一定长度的随机字符串
+     *
+     * @param length 指定字符串长度
+     * @return 一定长度的字符串
+     */
+    public static String getRandomStringByLength(int length) {
+        String base = "abcdefghijklmnopqrstuvwxyz0123456789";
+        Random random = new Random();
+        StringBuffer sb = new StringBuffer();
+        for (int i = 0; i < length; i++) {
+            int number = random.nextInt(base.length());
+            sb.append(base.charAt(number));
+        }
+        return sb.toString();
+    }
+
+    public static String generateTaskId() {
+        String yesterday = DateUtils.getYesterday_ymd();
+        return yesterday + "_" + getRandomStringByLength(6);
+    }
+}

src/main/java/com/winhc/bigdata/task/util/JsonUtils.java

@@ -0,0 +1,19 @@
+package com.winhc.bigdata.task.util;
+
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/6/22 14:18
+ * @Description:
+ */
+public class JsonUtils {
+    public static String jsonObjToString(Object jsonObject) {
+        return JSONObject.toJSONString(jsonObject, SerializerFeature.WriteMapNullValue);
+    }
+
+    public static String jsonObjToStringNotWriteMapNull(Object jsonObject) {
+        return JSONObject.toJSONString(jsonObject, SerializerFeature.IgnoreNonFieldGetter);
+    }
+}

src/main/java/com/winhc/bigdata/task/util/MaxBatchQueryUtils.java

@@ -0,0 +1,50 @@
+package com.winhc.bigdata.task.util;
+
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * @author: XuJiakai
+ * 2021/5/28 17:05
+ * 超长terms 查询
+ * 突破1024限制
+ */
+public class MaxBatchQueryUtils {
+    public static BoolQueryBuilder addTerms(List<String> ids, String key) {
+        return addTerms(new HashSet<>(ids), key);
+    }
+
+    public static BoolQueryBuilder addTerms(Set<String> ids, String key) {
+        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+        List<List<String>> lists = fixedGrouping(new ArrayList<>(ids), 1024);
+        for (List<String> list : lists) {
+            boolQuery.should(QueryBuilders.termsQuery(key, list));
+        }
+        return boolQuery;
+    }
+
+
+    private static <T> List<List<T>> fixedGrouping(List<T> source, int n) {
+        if (null == source || source.size() == 0 || n <= 0)
+            return null;
+        List<List<T>> result = new ArrayList<>();
+        int remainder = source.size() % n;
+        int size = (source.size() / n);
+        for (int i = 0; i < size; i++) {
+            List<T> subset = null;
+            subset = source.subList(i * n, (i + 1) * n);
+            result.add(subset);
+        }
+        if (remainder > 0) {
+            List<T> subset = null;
+            subset = source.subList(size * n, size * n + remainder);
+            result.add(subset);
+        }
+        return result;
+    }
+}

src/main/java/com/winhc/bigdata/task/util/OkHttpUtils.java

@@ -0,0 +1,42 @@
+package com.winhc.bigdata.task.util;
+
+import lombok.AllArgsConstructor;
+import okhttp3.*;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/7 11:54
+ * @Description:
+ */
+@Component
+@AllArgsConstructor
+public class OkHttpUtils {
+    private OkHttpClient client;
+    private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8");
+
+    public String post(String url, String json) throws IOException {
+        RequestBody body = RequestBody.create(JSON, json);
+        try (Response response = client.newCall((new Request.Builder()).url(url).post(body).build()).execute()) {
+            return parseBody(response);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return "";
+        }
+    }
+
+    public String get(String url) throws IOException {
+        try (Response response = client.newCall(new Request.Builder().url(url).get().build()).execute()) {
+            return parseBody(response);
+        }
+    }
+
+    private static String parseBody(Response response) throws IOException {
+        return response == null ? "" : response.isSuccessful() ? Objects.requireNonNull(response.body()).string() : "";
+    }
+
+
+}

src/main/java/com/winhc/bigdata/task/util/SpringContextUtil.java

@@ -0,0 +1,46 @@
+package com.winhc.bigdata.task.util;
+
+
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Component;
+
+
+/**
+ * @author: XuJiakai
+ * 2021/5/24 15:22
+ */
+@Component
+public class SpringContextUtil implements ApplicationContextAware {
+
+    /*
+      Spring应用上下文环境
+     */
+    private static ApplicationContext applicationContext;
+
+    /**
+     * 实现ApplicationContextAware接口的回调方法，设置上下文环境
+     */
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext)
+            throws BeansException {
+        SpringContextUtil.applicationContext = applicationContext;
+    }
+
+    public static ApplicationContext getApplicationContext() {
+        return applicationContext;
+    }
+
+    /**
+     * 获取对象 这里重写了bean方法，起主要作用
+     * example: getBean("userService")//注意： 类名首字母一定要小写！
+     */
+    public static Object getBean(String beanId) throws BeansException {
+        return applicationContext.getBean(beanId);
+    }
+
+    public static <T> T getBean(Class<T> requiredType) throws BeansException {
+        return applicationContext.getBean(requiredType);
+    }
+}