maven工具及logback
舟率率 12/29/2021 logbackmaven
# 打包命令
mvn clean package -Dmaven.test.skip=true
1
# 卸载本地仓库包
mvn dependency:purge-local-repository -DmanualInclude="groupId:artifactId"
1
# 发布到私有仓库
mvn clean deploy -Dmaven.test.skip=true
1
# pom.xml
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.compilerVersion>1.8</maven.compiler.compilerVersion>
<flink.version>1.12.0</flink.version>
<spark.version>3.1.1</spark.version> <!--k8s 3.3.0 local 2.4.0-->
<hive.version>3.1.0</hive.version>
<kafka.version>2.5.0</kafka.version>
<scala.version>2.11</scala.version>
<hadoop.version>3.1.1</hadoop.version>
<apache-phoenix-version>4.14.2-HBase-1.4</apache-phoenix-version>
<scope.type.test>test</scope.type.test>
<scope.type.compile>compile</scope.type.compile>
<scope.type.provided>provided</scope.type.provided>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<apollo.version>1.5.1</apollo.version>
<jedis.version>3.3.0</jedis.version>
<lombok.version>1.18.10</lombok.version>
<logback.classic.version>1.2.9</logback.classic.version>
<guava.version>30.1.1-jre</guava.version>
<elasticsearch.version>6.7.0</elasticsearch.version>
<dingtalk.version>1.0.1</dingtalk.version>
<commons.dbutils.version>1.6</commons.dbutils.version>
<mysql.connector.version>8.0.29</mysql.connector.version>
<tdengine.connector.version>3.2.2</tdengine.connector.version>
<hikaricp.version>4.0.3</hikaricp.version>
<jpmml.version>1.4.13</jpmml.version>
<scala.logging.version>3.9.2</scala.logging.version>
<logback.kafka.appender.version>0.2.0-RC2</logback.kafka.appender.version>
<junit.version>4.13.1</junit.version>
<flink.jackson.version>2.10.1-12.0</flink.jackson.version>
<jackson.version>2.14.3</jackson.version>
<okhttp.version>3.14.9</okhttp.version>
<parquet.avro.version>1.11.1</parquet.avro.version>
<avro.version>1.10.0</avro.version>
<httpclient.version>4.5.13</httpclient.version>
<httpasyncclient.version>4.1.3</httpasyncclient.version>
<httpcore.nio.version>4.4.13</httpcore.nio.version>
<httpcore.version>4.4.13</httpcore.version>
<findbugs.version>3.0.2</findbugs.version>
<protobuf.version>3.19.4</protobuf.version>
<okio.version>1.17.2</okio.version>
<nacos.version>2.1.2</nacos.version>
<codec.version>1.15</codec.version>
<commons.cli.version>1.4</commons.cli.version>
<aws.version>1.11.788</aws.version>
<scala.maven.plugin.version>4.0.2</scala.maven.plugin.version>
<spark.scala.version>2.12</spark.scala.version>
<scala.lib.version>2.12.18</scala.lib.version>
<spark.jackson.version>2.10.1</spark.jackson.version>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>
<encoding>${project.build.sourceEncoding}</encoding>
</configuration>
</plugin>
<!-- scala编译插件 mixed Java/Scala projects, see https://davidb.github.io/scala-maven-plugin/example_java.html -->
<plugin>
<!-- see http://davidb.github.com/scala-maven-plugin -->
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.1</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${scala.lib.version}</scalaVersion>
<args>
<arg>-target:jvm-1.8</arg>
</args>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<!-- 去掉dependency-reduced-pom.xml -->
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<!--项目中,存在一个jar包,版本不同,还必须保留两种版本-->
<!--举例:flink cep和cdc都需要guava,但一个需要18版本,一个需要30版本-->
<!--这个表示会生成两个jar包，一个原始的，一个shade的-->
<shadedArtifactAttached>true</shadedArtifactAttached>
<!--这个表示生成的shade的包 它的后缀名称是什么，通过这个后缀名称，在引用的时候的，就不会出现引用了shade包的情况了。 -->
<shadedClassifierName>with-guava-${flink.version}</shadedClassifierName>
<relocations>
<relocation>
<pattern>org.apache.flink.shaded.guava18.com.google</pattern>
<shadedPattern>hidden.org.apache.flink.shaded.guava18.com.google</shadedPattern>
<includes>
<include>org.apache.flink.shaded.guava18.com.google.*</include>
</includes>
</relocation>
</relocations>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>module-info.class</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>core-site.xml</exclude>
<exclude>hive-site.xml</exclude>
<exclude>hdfs-site.xml</exclude>
</excludes>
</filter>
<filter>
<!--这里重写了 tomcat-embed-core 包的 Tomcat 类-->
<!--就让 tomcat-embed-core 包的 Tomcat 类在打包时排除，让自己写的不被覆盖-->
<artifact>org.apache.flink:flink-connector-jdbc_2.11</artifact>
<excludes>
<exclude>org/apache/flink/connector/jdbc/dialect/AbstractDialect.java</exclude>
<exclude>org/apache/flink/connector/jdbc/dialect/JdbcDialects.java</exclude>
<exclude>
org/apache/flink/connector/jdbc/internal/converter/AbstractJdbcRowConverter.java
</exclude>
</excludes>
</filter>
</filters>
<finalName>${project.artifactId}-jar-with-dependencies</finalName>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>reference.conf</mainClass>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<artifactSet>
<excludes>
<exclude>org.slf4j:slf4j-simple</exclude>
<exclude>org.slf4j:slf4j-log4j12</exclude>
<exclude>log4j:log4j</exclude>
<exclude>org.apache.logging.log4j:log4j-api</exclude>
<exclude>org.apache.logging.log4j:log4j-core</exclude>
<exclude>org.apache.logging.log4j:log4j-slf4j-impl</exclude>
</excludes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<!--若需要发布到私服，则需要添加-->
<distributionManagement>
<snapshotRepository>
<id>nexus-snapshots</id>
<name>Snapshot repository</name>
<url>私服地址</url>
<!--<url>http://ip:8082/repository/maven-snapshots/</url>-->
</snapshotRepository>
<repository>
<id>nexus-releases</id>
<name>Release repository</name>
<url>私服地址</url>
<!--<url>http://ip:8082/repository/maven-releases/</url>-->
</repository>
</distributionManagement>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
# setting.xml
MAVEN配置之SETTING配置详解 (opens new window)
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<localRepository>D:/mvn/localRepository</localRepository>
<!--私服，若没有，可不写-->
<servers>
<server>
<id>nexus-releases</id>
<username>用户名</username>
<password>密码</password>
</server>
<server>
<id>nexus-snapshots</id>
<username>用户名</username>
<password>密码</password>
</server>
</servers>
<mirrors>
<mirror>
<id>aliyun-maven</id>
<mirrorOf>central</mirrorOf>
<name>阿里云公共仓库</name>
<url>https://maven.aliyun.com/repository/public</url>
</mirror>
<mirror>
<id>aliyun-spring</id>
<mirrorOf>spring</mirrorOf>
<name>阿里云Spring仓库</name>
<url>https://maven.aliyun.com/repository/spring</url>
</mirror>
<mirror>
<id>nexus-public</id>
<mirrorOf>*,!central,!spring,!cloudera</mirrorOf>
<url>私服地址</url>
<!--<url>http://ip:8082/repository/maven-public/</url>-->
</mirror>
</mirrors>
<profiles>
<profile>
<id>aliyun</id>
<repositories>
<repository>
<id>aliyun-maven</id>
<name>aliyun-maven</name>
<url>https://maven.aliyun.com/repository/public</url>
<layout>default</layout>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>spring</id>
<url>https://maven.aliyun.com/repository/spring</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>apache-snapshots</id>
<url>https://maven.aliyun.com/repository/apache-snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>spring-plugin</id>
<url>https://maven.aliyun.com/repository/spring-plugin</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
</pluginRepositories>
</profile>
<profile>
<id>cloudera</id>
<repositories>
<repository>
<id>cloudera</id>
<name>cloudera</name>
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
<layout>default</layout>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
</profile>
<profile>
<id>nexus</id>
<repositories>
<repository>
<id>nexus-public</id>
<url>私服地址</url>
<!--<url>http://ip:8082/repository/maven-public/</url>-->
<releases>
<enabled>true</enabled>
<updatePolicy>never</updatePolicy>
</releases>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
</snapshots>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>nexus-public</id>
<url>私服地址</url>
<!--<url>http://ip:8082/repository/maven-public/</url>-->
<releases>
<enabled>true</enabled>
<updatePolicy>never</updatePolicy>
</releases>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
</snapshots>
</pluginRepository>
</pluginRepositories>
</profile>
</profiles>
<activeProfiles>
<activeProfile>aliyun</activeProfile>
<activeProfile>cloudera</activeProfile>
<activeProfile>nexus</activeProfile>
</activeProfiles>
</settings>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
# logback
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<property name="kafka.topic" value="test_topic"/>
<property name="bootstrap.servers" value="test_servers"/>
<!-- 属性文件:在properties文件中找到对应的配置项 -->
<property name="logging.path" value="logs"/>
<!-- 输出格式 -->
<property name="out.pattern" value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5p] [%t] [%c][%L] - %m%n"/>
<!-- 活动文件的大小 -->
<property name="max.file.size" value="100MB"/>
<!-- 保留的归档文件的最大数量 -->
<property name="max.history" value="7"/>
<!-- 控制所有归档日志文件的总大小 -->
<property name="total.size.cap" value="10GB"/>
<!--屏蔽某个包或类下的日志-->
<logger name="org.apache.kafka" level="DEBUG"/>
<logger name="com.eharmony.pho.hbase.query.PhoenixHBaseQueryExecutor" level="ERROR"/>
<logger name="com.eharmony.pho.hbase.query.PhoenixHBaseQueryExecutor" level="ERROR">
<appender-ref ref="FILE"/>
</logger>
<conversionRule conversionWord="cl" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
<!-- 控制台输出日志 -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> <!---->
<withJansi>false</withJansi>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss}|%thread|%-5level|%logger{80}|%method|%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<append>true</append>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>DEBUG</level>
</filter>
<file>
${logging.path}/log.log
</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- 每隔半天：.yyyy-MM-dd-a;每小时：.yyyy-MM-dd-HH;每分钟：.yyyy-MM-dd-HH-mm -->
<FileNamePattern>${logging.path}/log.%d{yyyy-MM-dd-HH-mm}.%i.log</FileNamePattern>
<!-- each file should be at most 100MB, keep 60 days worth of history, but at most 20GB -->
<!-- 控制归档文件的最大数量的保存，删除旧的文件，默认单位天数 -->
<maxFileSize>${max.file.size}</maxFileSize>
<maxHistory>${max.history}</maxHistory>
<totalSizeCap>${total.size.cap}</totalSizeCap>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>${out.pattern}</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!--使用kafkaAppender时,依赖不可以重复slf4j,只需要依赖logback-kafka-appender和logback-classic即可-->
<appender name="kafkaAppender" class="com.github.danielwegener.logback.kafka.KafkaAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
<topic>${kafka.topic}</topic>
<keyingStrategy class="com.github.danielwegener.logback.kafka.keying.NoKeyKeyingStrategy"/>
<deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy"/>
<producerConfig>acks=all</producerConfig>
<!--生产者可以用来缓冲等待发送到服务器的记录的总内存字节-->
<producerConfig>buffer.memory=33554432</producerConfig>
<!--每个消息集合都会经过GZIP压缩，能够很好地节省网络传输带宽和Kafka Broker端的磁盘占用-->
<producerConfig>compression.type=gzip</producerConfig>
<!--请求的最大大小(以字节为单位)-->
<producerConfig>max.request.size=2097152</producerConfig>
<!--生产者将在请求传输之间到达的所有记录归为一个批处理的请求-->
<producerConfig>linger.ms=1000</producerConfig>
<producerConfig>max.block.ms=1000</producerConfig>
<producerConfig>batch.size=16384</producerConfig>
<!-- bootstrap.servers is the only mandatory producerConfig -->
<producerConfig>bootstrap.servers=${bootstrap.servers}</producerConfig>
<!-- this is the fallback appender if kafka is not available. -->
<appender-ref ref="STDOUT"/>
</appender>
<!--异步输出-->
<appender name="ASYNC_KAFKA" class="ch.qos.logback.classic.AsyncAppender">
<appender-ref ref="kafkaAppender"/>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<appender-ref ref="FILE"/>
</appender>
<root level="DEBUG">
<appender-ref ref="STDOUT"/>
</root>
<!--kafkaAppender无法使用DEBUG级别-->
<root level="INFO">
<appender-ref ref="ASYNC_FILE"/>
<!--<appender-ref ref="ASYNC_KAFKA"/>-->
</root>
</configuration>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
# 依赖
<!--logback.xml,自带slf4j-api-->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback.classic.version}</version>
</dependency>
<!--logback没有配置发送kafka,则不需要此包-->
<dependency>
<groupId>com.github.danielwegener</groupId>
<artifactId>logback-kafka-appender</artifactId>
<version>${logback.kafka.appender.version}</version>
</dependency>
1
2
3
4
5
6
7
8
9
10
11
12
2
3
4
5
6
7
8
9
10
11
12
# 屏蔽某个包或类下的日志
<logger name="com.eharmony.pho.hbase.query.PhoenixHBaseQueryExecutor" level="ERROR"/>
1
2
2