You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
340 lines
13 KiB
340 lines
13 KiB
<?xml version="1.0" encoding="UTF-8"?>
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<groupId>com.freesun</groupId>
|
|
<artifactId>etl</artifactId>
|
|
<version>1.0-SNAPSHOT</version>
|
|
<modules>
|
|
<module>flink2iceberg</module>
|
|
<module>flink2hudi</module>
|
|
</modules>
|
|
<packaging>pom</packaging>
|
|
|
|
<name>FS-IOT ETL</name>
|
|
<url>http://www.free-sun.com.cn/</url>
|
|
|
|
<properties>
|
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
|
<flink.version>1.11.2</flink.version>
|
|
<!--<hadoop.version>2.7.3</hadoop.version>-->
|
|
<hadoop.version>2.8.2</hadoop.version>
|
|
<hadoop.scope>compile</hadoop.scope>
|
|
<scala.binary.version>2.11</scala.binary.version>
|
|
<scala.version>2.11.11</scala.version>
|
|
<jackson.version>2.12.4</jackson.version>
|
|
</properties>
|
|
|
|
<profiles>
|
|
<profile>
|
|
<id>dev</id>
|
|
<activation>
|
|
<property>
|
|
<name>idea.version</name>
|
|
</property>
|
|
</activation>
|
|
<properties>
|
|
<profiles.active>development</profiles.active>
|
|
<flink.scope>compile</flink.scope>
|
|
<scala.scope>compile</scala.scope>
|
|
</properties>
|
|
</profile>
|
|
<profile>
|
|
<id>prod</id>
|
|
<properties>
|
|
<profiles.active>production</profiles.active>
|
|
<flink.scope>provided</flink.scope>
|
|
<scala.scope>provided</scala.scope>
|
|
</properties>
|
|
</profile>
|
|
</profiles>
|
|
|
|
<dependencies>
|
|
<!-- Apache Flink dependencies -->
|
|
<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
|
|
<dependency>
|
|
<groupId>org.apache.flink</groupId>
|
|
<artifactId>flink-scala_${scala.binary.version}</artifactId>
|
|
<version>${flink.version}</version>
|
|
<scope>${flink.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.flink</groupId>
|
|
<artifactId>flink-clients_${scala.binary.version}</artifactId>
|
|
<version>${flink.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.flink</groupId>
|
|
<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
|
|
<version>${flink.version}</version>
|
|
<scope>${flink.scope}</scope>
|
|
</dependency>
|
|
|
|
<!-- Scala Library, provided by Flink as well. -->
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-library</artifactId>
|
|
<version>${scala.version}</version>
|
|
<scope>${scala.scope}</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-compiler</artifactId>
|
|
<version>${scala.version}</version>
|
|
<scope>${scala.scope}</scope>
|
|
</dependency>
|
|
|
|
<!-- Add connector dependencies here. They must be in the default scope (compile). -->
|
|
<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-kafka -->
|
|
<dependency>
|
|
<groupId>org.apache.flink</groupId>
|
|
<artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
|
|
<version>${flink.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kafka</groupId>
|
|
<artifactId>kafka-clients</artifactId>
|
|
<version>2.0.1</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.flink</groupId>
|
|
<artifactId>flink-connector-elasticsearch6_${scala.binary.version}</artifactId>
|
|
<version>${flink.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.elasticsearch.client</groupId>
|
|
<artifactId>elasticsearch-rest-high-level-client</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<!-- ****Apache Flink dependencies END**** -->
|
|
|
|
<dependency>
|
|
<groupId>free-sun.com.cn</groupId>
|
|
<artifactId>comm_utils</artifactId>
|
|
<version>3.0</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-databind</artifactId>
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
|
|
<dependency>
|
|
<groupId>org.postgresql</groupId>
|
|
<artifactId>postgresql</artifactId>
|
|
<version>42.1.1</version>
|
|
</dependency>
|
|
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-dbcp2 -->
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-dbcp2</artifactId>
|
|
<version>2.1.1</version>
|
|
</dependency>
|
|
|
|
<!--joda-time kyro serializer-->
|
|
<dependency>
|
|
<groupId>de.javakaffee</groupId>
|
|
<artifactId>kryo-serializers</artifactId>
|
|
<version>0.42</version>
|
|
</dependency>
|
|
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-joda -->
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.datatype</groupId>
|
|
<artifactId>jackson-datatype-joda</artifactId>
|
|
<version>2.9.4</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-databind</artifactId>
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-databind</artifactId>
|
|
<version>${jackson.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
<version>${jackson.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
<version>${jackson.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.module</groupId>
|
|
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
|
|
<version>${jackson.version}</version>
|
|
</dependency>
|
|
|
|
<!-- junit-->
|
|
<dependency>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
<version>4.12</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
|
|
|
|
<build>
|
|
|
|
<filters>
|
|
<filter>../src/main/resources/${profiles.active}/conf.properties</filter>
|
|
</filters>
|
|
|
|
<resources>
|
|
<resource>
|
|
<directory>../src/main/resources</directory>
|
|
<filtering>true</filtering>
|
|
<includes>
|
|
<include>*.properties</include>
|
|
</includes>
|
|
</resource>
|
|
<resource>
|
|
<!--Wrong encoding after activating resource filtering-->
|
|
<directory>../src/main/resources</directory>
|
|
<filtering>false</filtering>
|
|
<excludes>
|
|
<exclude>*.properties</exclude>
|
|
</excludes>
|
|
</resource>
|
|
</resources>
|
|
|
|
<plugins>
|
|
<!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
|
|
<!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-shade-plugin</artifactId>
|
|
<version>3.0.0</version>
|
|
<executions>
|
|
<!-- Run shade goal on package phase -->
|
|
<execution>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>shade</goal>
|
|
</goals>
|
|
<configuration>
|
|
<artifactSet>
|
|
<excludes>
|
|
<exclude>org.apache.flink:force-shading</exclude>
|
|
<exclude>com.google.code.findbugs:jsr305</exclude>
|
|
<exclude>org.slf4j:*</exclude>
|
|
<exclude>log4j:*</exclude>
|
|
</excludes>
|
|
</artifactSet>
|
|
<filters>
|
|
<filter>
|
|
<!-- Do not copy the signatures in the META-INF folder.
|
|
Otherwise, this might cause SecurityExceptions when using the JAR. -->
|
|
<artifact>*:*</artifact>
|
|
<excludes>
|
|
<exclude>META-INF/*.SF</exclude>
|
|
<exclude>META-INF/*.DSA</exclude>
|
|
<exclude>META-INF/*.RSA</exclude>
|
|
</excludes>
|
|
</filter>
|
|
</filters>
|
|
<transformers>
|
|
<transformer
|
|
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
|
|
<resource>reference.conf</resource>
|
|
</transformer>
|
|
<transformer
|
|
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
|
<mainClass>com.freesun.alarm.StreamingJob</mainClass>
|
|
</transformer>
|
|
</transformers>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<!-- Java Compiler -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-compiler-plugin</artifactId>
|
|
<version>3.1</version>
|
|
<configuration>
|
|
<source>1.8</source>
|
|
<target>1.8</target>
|
|
</configuration>
|
|
</plugin>
|
|
|
|
<!-- Scala Compiler -->
|
|
<plugin>
|
|
<groupId>net.alchim31.maven</groupId>
|
|
<artifactId>scala-maven-plugin</artifactId>
|
|
<version>3.2.2</version>
|
|
<executions>
|
|
<execution>
|
|
<goals>
|
|
<goal>compile</goal>
|
|
<goal>testCompile</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
|
|
<distributionManagement>
|
|
<repository>
|
|
<id>fs-releases</id>
|
|
<name>fs-releases</name>
|
|
<url>http://10.8.30.22:8081/repository/fs-releases/</url>
|
|
</repository>
|
|
</distributionManagement>
|
|
|
|
<repositories>
|
|
<repository>
|
|
<id>apache.snapshots</id>
|
|
<name>Apache Development Snapshot Repository</name>
|
|
<url>https://repository.apache.org/content/repositories/snapshots/</url>
|
|
<releases>
|
|
<enabled>false</enabled>
|
|
</releases>
|
|
<snapshots>
|
|
<enabled>true</enabled>
|
|
</snapshots>
|
|
</repository>
|
|
</repositories>
|
|
</project>
|