datax入门(data-web的简单使用)——02

CSDN 2024-08-07 13:33:03 阅读 89

datax入门(data-web的简单使用)——02

1. 前言1.1 关于data-web官网1.1.1 源码下载1.1.2 datax-Web部署手册1.1.2.1 Linux环境部署手册1.1.2.2 本地开发环境部署手册

1.2 关于datax入门

2. 下载之后打包、启动、登录2.1 我的本地环境2.2 修改配置2.3 初始化sql脚本、启动2.4 登录

3. 其他说明3.1 python安装3.2 问题

1. 前言

1.1 关于data-web官网

1.1.1 源码下载

https://gitee.com/liuersusu/datax-web.

1.1.2 datax-Web部署手册

1.1.2.1 Linux环境部署手册

如下:

https://github.com/WeiYe-Jing/datax-web/blob/master/doc/datax-web/datax-web-deploy.md.

在这里插入图片描述

1.1.2.2 本地开发环境部署手册

如下:

https://github.com/WeiYe-Jing/datax-web/blob/master/userGuid.md.

1.2 关于datax入门

2. 下载之后打包、启动、登录

2.1 我的本地环境

JDK1.8,maven3.9.0,python3.11,mysql8

2.2 修改配置

如下:

在这里插入图片描述

datax-admin->application.yml如下:

<code>server:

port: 8080

# port: ${server.port}

spring:

#数据源

# datasource:

# # username: root

# #password: root

# #url: jdbc:mysql://localhost:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8

# password: ${DB_PASSWORD:password}

# username: ${DB_USERNAME:username}

# url: jdbc:mysql://${DB_HOST:127.0.0.1}:${DB_PORT:3306}/${DB_DATABASE:dataxweb}?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8

# driver-class-name: com.mysql.jdbc.Driver

datasource:

# username: root

#password: root

#url: jdbc:mysql://localhost:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8

username: root

password: susu@123

url: jdbc:mysql://127.0.0.1:3306/datax_1?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8

driver-class-name: com.mysql.jdbc.Driver

hikari:

## 最小空闲连接数量

minimum-idle: 5

## 空闲连接存活最大时间,默认600000(10分钟)

idle-timeout: 180000

## 连接池最大连接数,默认是10

maximum-pool-size: 10

## 数据库连接超时时间,默认30秒,即30000

connection-timeout: 30000

connection-test-query: SELECT 1

##此属性控制池中连接的最长生命周期,值0表示无限生命周期,默认1800000即30分钟

max-lifetime: 1800000

# datax-web email

mail:

host: smtp.qq.com

port: 25

username: xxx@qq.com

password: xxx

# username: ${mail.username}

# password: ${mail.password}

properties:

mail:

smtp:

auth: true

starttls:

enable: true

required: true

socketFactory:

class: javax.net.ssl.SSLSocketFactory

management:

health:

mail:

enabled: false

server:

servlet:

context-path: /actuator

mybatis-plus:

# mapper.xml文件扫描

mapper-locations: classpath*:/mybatis-mapper/*Mapper.xml

# 实体扫描,多个package用逗号或者分号分隔

#typeAliasesPackage: com.yibo.essyncclient.*.entity

global-config:

# 数据库相关配置

db-config:

# 主键类型 AUTO:"数据库ID自增", INPUT:"用户输入ID", ID_WORKER:"全局唯一ID (数字类型唯一ID)", UUID:"全局唯一ID UUID";

id-type: AUTO

# 字段策略 IGNORED:"忽略判断",NOT_NULL:"非 NULL 判断"),NOT_EMPTY:"非空判断"

field-strategy: NOT_NULL

# 驼峰下划线转换

column-underline: true

# 逻辑删除

logic-delete-value: 0

logic-not-delete-value: 1

# 数据库类型

db-type: mysql

banner: false

# mybatis原生配置

configuration:

map-underscore-to-camel-case: true

cache-enabled: false

call-setters-on-nulls: true

jdbc-type-for-null: 'null'

type-handlers-package: com.wugui.datax.admin.core.handler

# 配置mybatis-plus打印sql日志

logging:

level:

com.wugui.datax.admin.mapper: info

path: ./data/applogs/admin

# level:

# com.wugui.datax.admin.mapper: error

# path: ${data.path}/applogs/admin

#datax-job, access token

datax:

job:

accessToken:

#i18n (default empty as chinese, "en" as english)

i18n:

## triggerpool max size

triggerpool:

fast:

max: 200

slow:

max: 100

### log retention days

logretentiondays: 30

datasource:

aes:

key: AD42F6697B035B75

datax-executor–>application.yml

# web port

server:

# port: ${server.port}

port: 8081

# log config

logging:

config: classpath:logback.xml

# path: ${data.path}/applogs/executor/jobhandler

path: ./data/applogs/executor/jobhandler

datax:

job:

admin:

### datax admin address list, such as "http://address" or "http://address01,http://address02"

addresses: http://127.0.0.1:8080

# addresses: http://127.0.0.1:${datax.admin.port}

executor:

appname: datax-executor

ip:

port: 9999

# port: ${executor.port:9999}

### job log path

logpath: ./data/applogs/executor/jobhandler

# logpath: ${data.path}/applogs/executor/jobhandler

### job log retention days

logretentiondays: 30

### job, access token

accessToken:

executor:

jsonpath: /Users/susu/datax_use/data/applogs

# jsonpath: D:\\temp\\executor\\json\\

# jsonpath: ${json.path}

pypath: /Users/susu/study_down/about_datax/datax/bin/datax.py

# pypath: F:\tools\datax\bin\datax.py

# pypath: ${python.path}

2.3 初始化sql脚本、启动

下载的项目里有datax_web.sql文件,执行就行:

在这里插入图片描述

<code>/*

Navicat Premium Data Transfer

Source Server : localhost

Source Server Type : MySQL

Source Server Version : 50725

Source Host : localhost:3306

Source Schema : datax_web

Target Server Type : MySQL

Target Server Version : 50725

File Encoding : 65001

Date: 15/12/2019 22:27:10

*/

SET NAMES utf8mb4;

SET FOREIGN_KEY_CHECKS = 0;

-- ----------------------------code>

-- Table structure for job_group

-- ----------------------------

DROP TABLE IF EXISTS `job_group`;

CREATE TABLE `job_group` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`app_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '执行器AppName',

`title` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '执行器名称',

`order` int(11) NOT NULL DEFAULT 0 COMMENT '排序',

`address_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT '执行器地址类型:0=自动注册、1=手动录入',

`address_list` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器地址列表,多地址逗号分隔',

PRIMARY KEY (`id`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

-- ----------------------------

-- Records of job_group

-- ----------------------------

INSERT INTO `job_group` VALUES (1, 'datax-executor', 'datax执行器', 1, 0, NULL);

-- ----------------------------

-- Table structure for job_info

-- ----------------------------

DROP TABLE IF EXISTS `job_info`;

CREATE TABLE `job_info` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`job_group` int(11) NOT NULL COMMENT '执行器主键ID',

`job_cron` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '任务执行CRON',

`job_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,

`add_time` datetime(0) NULL DEFAULT NULL,

`update_time` datetime(0) NULL DEFAULT NULL,

`author` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '作者',

`alarm_email` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '报警邮件',

`executor_route_strategy` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器路由策略',

`executor_handler` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器任务handler',

`executor_param` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器任务参数',

`executor_block_strategy` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '阻塞处理策略',

`executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT '任务执行超时时间,单位秒',

`executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数',

`glue_type` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'GLUE类型',

`glue_source` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT 'GLUE源代码',

`glue_remark` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'GLUE备注',

`glue_updatetime` datetime(0) NULL DEFAULT NULL COMMENT 'GLUE更新时间',

`child_jobid` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '子任务ID,多个逗号分隔',

`trigger_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '调度状态:0-停止,1-运行',

`trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上次调度时间',

`trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '下次调度时间',

`job_json` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT 'datax运行脚本',

PRIMARY KEY (`id`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 7 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

-- ----------------------------

-- Table structure for job_jdbc_datasource

-- ----------------------------

DROP TABLE IF EXISTS `job_jdbc_datasource`;

CREATE TABLE `job_jdbc_datasource` (

`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '自增主键',

`datasource_name` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '数据源名称',

`datasource_group` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT 'Default' COMMENT '数据源分组',

`jdbc_username` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '用户名',

`jdbc_password` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '密码',

`jdbc_url` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'jdbc url',

`jdbc_driver_class` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'jdbc驱动类',

`status` tinyint(1) NOT NULL DEFAULT 1 COMMENT '状态:0删除 1启用 2禁用',

`create_by` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '创建人',

`create_date` datetime(0) NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT '创建时间',

`update_by` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '更新人',

`update_date` datetime(0) NULL DEFAULT NULL COMMENT '更新时间',

`comments` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '备注',

PRIMARY KEY (`id`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 6 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = 'jdbc数据源配置' ROW_FORMAT = Dynamic;

-- ----------------------------

-- Table structure for job_lock

-- ----------------------------

DROP TABLE IF EXISTS `job_lock`;

CREATE TABLE `job_lock` (

`lock_name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '锁名称',

PRIMARY KEY (`lock_name`) USING BTREE

) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

-- ----------------------------

-- Records of job_lock

-- ----------------------------

INSERT INTO `job_lock` VALUES ('schedule_lock');

-- ----------------------------

-- Table structure for job_log

-- ----------------------------

DROP TABLE IF EXISTS `job_log`;

CREATE TABLE `job_log` (

`id` bigint(20) NOT NULL AUTO_INCREMENT,

`job_group` int(11) NOT NULL COMMENT '执行器主键ID',

`job_id` int(11) NOT NULL COMMENT '任务,主键ID',

`job_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,

`executor_address` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器地址,本次执行的地址',

`executor_handler` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器任务handler',

`executor_param` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器任务参数',

`executor_sharding_param` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器任务分片参数,格式如 1/2',

`executor_fail_retry_count` int(11) NULL DEFAULT 0 COMMENT '失败重试次数',

`trigger_time` datetime(0) NULL DEFAULT NULL COMMENT '调度-时间',

`trigger_code` int(11) NOT NULL COMMENT '调度-结果',

`trigger_msg` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT '调度-日志',

`handle_time` datetime(0) NULL DEFAULT NULL COMMENT '执行-时间',

`handle_code` int(11) NOT NULL COMMENT '执行-状态',

`handle_msg` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT '执行-日志',

`alarm_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '告警状态:0-默认、1-无需告警、2-告警成功、3-告警失败',

`process_id` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'datax进程Id',

`max_id` bigint(20) NULL DEFAULT NULL COMMENT '增量表max id',

PRIMARY KEY (`id`) USING BTREE,

INDEX `I_trigger_time`(`trigger_time`) USING BTREE,

INDEX `I_handle_code`(`handle_code`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 0 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

-- ----------------------------

-- Table structure for job_log_report

-- ----------------------------

DROP TABLE IF EXISTS `job_log_report`;

CREATE TABLE `job_log_report` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`trigger_day` datetime(0) NULL DEFAULT NULL COMMENT '调度-时间',

`running_count` int(11) NOT NULL DEFAULT 0 COMMENT '运行中-日志数量',

`suc_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行成功-日志数量',

`fail_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行失败-日志数量',

PRIMARY KEY (`id`) USING BTREE,

UNIQUE INDEX `i_trigger_day`(`trigger_day`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 28 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

-- ----------------------------

-- Records of job_log_report

-- ----------------------------

INSERT INTO `job_log_report` VALUES (20, '2019-12-07 00:00:00', 0, 0, 0);

INSERT INTO `job_log_report` VALUES (21, '2019-12-10 00:00:00', 77, 52, 23);

INSERT INTO `job_log_report` VALUES (22, '2019-12-11 00:00:00', 9, 2, 11);

INSERT INTO `job_log_report` VALUES (23, '2019-12-13 00:00:00', 9, 48, 74);

INSERT INTO `job_log_report` VALUES (24, '2019-12-12 00:00:00', 10, 8, 30);

INSERT INTO `job_log_report` VALUES (25, '2019-12-14 00:00:00', 78, 45, 66);

INSERT INTO `job_log_report` VALUES (26, '2019-12-15 00:00:00', 24, 76, 9);

INSERT INTO `job_log_report` VALUES (27, '2019-12-16 00:00:00', 23, 85, 10);

-- ----------------------------

-- Table structure for job_logglue

-- ----------------------------

DROP TABLE IF EXISTS `job_logglue`;

CREATE TABLE `job_logglue` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`job_id` int(11) NOT NULL COMMENT '任务,主键ID',

`glue_type` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'GLUE类型',

`glue_source` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT 'GLUE源代码',

`glue_remark` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'GLUE备注',

`add_time` datetime(0) NULL DEFAULT NULL,

`update_time` datetime(0) NULL DEFAULT NULL,

PRIMARY KEY (`id`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

-- ----------------------------

-- Table structure for job_registry

-- ----------------------------

DROP TABLE IF EXISTS `job_registry`;

CREATE TABLE `job_registry` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`registry_group` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,

`registry_key` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,

`registry_value` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,

`update_time` datetime(0) NULL DEFAULT NULL,

PRIMARY KEY (`id`) USING BTREE,

INDEX `i_g_k_v`(`registry_group`, `registry_key`, `registry_value`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 26 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

-- ----------------------------

-- Table structure for job_user

-- ----------------------------

DROP TABLE IF EXISTS `job_user`;

CREATE TABLE `job_user` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`username` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '账号',

`password` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '密码',

`role` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '角色:0-普通用户、1-管理员',

`permission` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '权限:执行器ID列表,多个逗号分割',

PRIMARY KEY (`id`) USING BTREE,

UNIQUE INDEX `i_username`(`username`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 10 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

-- ----------------------------

-- Records of job_user

-- ----------------------------

INSERT INTO `job_user` VALUES (1, 'admin', '$2a$10$2KCqRbra0Yn2TwvkZxtfLuWuUP5KyCWsljO/ci5pLD27pqR3TV1vy', 'ROLE_ADMIN', NULL);

/**

v2.1.1脚本更新

*/

ALTER TABLE `job_info`

ADD COLUMN `replace_param` VARCHAR(100) NULL DEFAULT NULL COMMENT '动态参数' AFTER `job_json`,

ADD COLUMN `jvm_param` VARCHAR(200) NULL DEFAULT NULL COMMENT 'jvm参数' AFTER `replace_param`,

ADD COLUMN `time_offset` INT(11) NULL DEFAULT '0'COMMENT '时间偏移量' AFTER `jvm_param`;

/**

增量改版脚本更新

*/

ALTER TABLE `job_info` DROP COLUMN `time_offset`;

ALTER TABLE `job_info`

ADD COLUMN `inc_start_time` DATETIME NULL DEFAULT NULL COMMENT '增量初始时间' AFTER `jvm_param`;

-- ----------------------------

-- Table structure for job_template

-- ----------------------------

DROP TABLE IF EXISTS `job_template`;

CREATE TABLE `job_template` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`job_group` int(11) NOT NULL COMMENT '执行器主键ID',

`job_cron` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '任务执行CRON',

`job_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,

`add_time` datetime(0) NULL DEFAULT NULL,

`update_time` datetime(0) NULL DEFAULT NULL,

`user_id` int(11) NOT NULL COMMENT '修改用户',

`alarm_email` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '报警邮件',

`executor_route_strategy` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器路由策略',

`executor_handler` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器任务handler',

`executor_param` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '执行器参数',

`executor_block_strategy` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '阻塞处理策略',

`executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT '任务执行超时时间,单位秒',

`executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数',

`glue_type` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'GLUE类型',

`glue_source` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL COMMENT 'GLUE源代码',

`glue_remark` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'GLUE备注',

`glue_updatetime` datetime(0) NULL DEFAULT NULL COMMENT 'GLUE更新时间',

`child_jobid` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '子任务ID,多个逗号分隔',

`trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上次调度时间',

`trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '下次调度时间',

`job_json` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT 'datax运行脚本',

`jvm_param` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'jvm参数',

`project_id` int(11) NULL DEFAULT NULL COMMENT '所属项目Id',

PRIMARY KEY (`id`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 22 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

/**

添加数据源字段

*/

ALTER TABLE `job_jdbc_datasource`

ADD COLUMN `datasource` VARCHAR(45) NOT NULL COMMENT '数据源' AFTER `datasource_name`;

/**

添加分区字段

*/

ALTER TABLE `job_info`

ADD COLUMN `partition_info` VARCHAR(100) NULL DEFAULT NULL COMMENT '分区信息' AFTER `inc_start_time`;

/**

2.1.1版本新增----------------------------------------------------------------------------------------------

*/

/**

最近一次执行状态

*/

ALTER TABLE `job_info`

ADD COLUMN `last_handle_code` INT(11) NULL DEFAULT '0' COMMENT '最近一次执行状态' AFTER `partition_info`;

/**

zookeeper地址

*/

ALTER TABLE `job_jdbc_datasource`

ADD COLUMN `zk_adress` VARCHAR(200) NULL DEFAULT NULL AFTER `jdbc_driver_class`;

ALTER TABLE `job_info`

CHANGE COLUMN `executor_timeout` `executor_timeout` INT(11) NOT NULL DEFAULT '0' COMMENT '任务执行超时时间,单位分钟' ;

/**

用户名密码改为非必填

*/

ALTER TABLE `job_jdbc_datasource`

CHANGE COLUMN `jdbc_username` `jdbc_username` VARCHAR(100) CHARACTER SET 'utf8mb4' NULL DEFAULT NULL COMMENT '用户名' ,

CHANGE COLUMN `jdbc_password` `jdbc_password` VARCHAR(200) CHARACTER SET 'utf8mb4' NULL DEFAULT NULL COMMENT '密码' ;

/**

添加mongodb数据库名字段

*/

ALTER TABLE `job_jdbc_datasource`

ADD COLUMN `database_name` VARCHAR(45) NULL DEFAULT NULL COMMENT '数据库名' AFTER `datasource_group`;

/**

添加执行器资源字段

*/

ALTER TABLE `job_registry`

ADD COLUMN `cpu_usage` DOUBLE NULL AFTER `registry_value`,

ADD COLUMN `memory_usage` DOUBLE NULL AFTER `cpu_usage`,

ADD COLUMN `load_average` DOUBLE NULL AFTER `memory_usage`;

-- ----------------------------

-- Table structure for job_permission

-- ----------------------------

DROP TABLE IF EXISTS `job_permission`;

CREATE TABLE `job_permission` (

`id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键',

`name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '权限名',

`description` varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '权限描述',

`url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,

`pid` int(11) NULL DEFAULT NULL,

PRIMARY KEY (`id`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

ALTER TABLE `job_info`

ADD COLUMN `replace_param_type` varchar(255) NULL COMMENT '增量时间格式' AFTER `last_handle_code`;

ALTER TABLE `job_info`

ADD COLUMN `project_id` int(11) NULL COMMENT '所属项目id' AFTER `job_desc`;

ALTER TABLE `job_info`

ADD COLUMN `reader_table` VARCHAR(255) NULL COMMENT 'reader表名称' AFTER `replace_param_type`,

ADD COLUMN `primary_key` VARCHAR(50) NULL COMMENT '增量表主键' AFTER `reader_table`,

ADD COLUMN `inc_start_id` VARCHAR(20) NULL COMMENT '增量初始id' AFTER `primary_key`,

ADD COLUMN `increment_type` TINYINT(4) NULL COMMENT '增量类型' AFTER `inc_start_id`,

ADD COLUMN `datasource_id` BIGINT(11) NULL COMMENT '数据源id' AFTER `increment_type`;

CREATE TABLE `job_project` (

`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',

`name` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT 'project name',

`description` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,

`user_id` int(11) NULL DEFAULT NULL COMMENT 'creator id',

`flag` tinyint(4) NULL DEFAULT 1 COMMENT '0 not available, 1 available',

`create_time` datetime(0) NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT 'create time',

`update_time` datetime(0) NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT 'update time',

PRIMARY KEY (`id`) USING BTREE

) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;

ALTER TABLE `job_info`

CHANGE COLUMN `author` `user_id` INT(11) NOT NULL COMMENT '修改用户' ;

ALTER TABLE `job_info`

CHANGE COLUMN `increment_type` `increment_type` TINYINT(4) NULL DEFAULT 0 COMMENT '增量类型' ;

2.4 登录

地址:

http://localhost:8080/index.html#/dashboard

默认管理员用户名:admin 密码:123456

在这里插入图片描述

3. 其他说明

3.1 python安装

通过官网下载安装

官网地址:https://www.python.org/downloads/.参考:python-01(入门基础篇).

通过这种方式安装会自动添加环境变量。 通过homebrew安装

3.2 问题

如果遇到下面的问题:

<code> Cannot run program "python": error=2, No such file or directory

在这里插入图片描述

解决问题:

如果安装的是python3,换一下命令,把<code>python换成python3(我的是Mac版的,其他的不清楚):

在这里插入图片描述

或者查看一下环境变量是否配置。



声明

本文内容仅代表作者观点,或转载于其他网站,本站不以此文作为商业用途
如有涉及侵权,请联系本站进行删除
转载本站原创文章,请注明来源及作者。