使用forkjoin批量插入数据库表

阳光穿透心脏的1/2处 2023-10-07 17:27 29阅读 0赞

### forkjoin基本使用 ###

链接: [多线程之Fork/Join使用][Fork_Join].

### 模拟导入数据 ###

import com.baomidou.mybatisplus.annotation.TableName;
    import com.baomidou.mybatisplus.annotation.IdType;
    import com.baomidou.mybatisplus.annotation.TableId;
    import java.io.Serializable;
    import lombok.Data;
    import lombok.EqualsAndHashCode;
    
    /**
     * <p>
     * 
     * </p>
     *
     * @author zhoum
     * @since 2022-01-30
     */
    @Data
    @EqualsAndHashCode(callSuper = false)
    @TableName("user_test")
    public class UserTestEntity implements Serializable {
        
    
        private static final long serialVersionUID = 1L;
    
        @TableId(value = "id", type = IdType.ASSIGN_ID)
        private String id;
    
        private String name1;
    
        private String name2;
    
        private String name3;
    
        private String name4;
    
        private String name5;
    
        private String name6;
    
        private String name7;
    
        private String name8;
    
        private String name9;
    
    
    }

package com.zm.forkjoin;
    
    import com.zm.entity.UserTestEntity;
    import com.zm.mapper.UserTestMapper;
    
    import java.util.ArrayList;
    import java.util.List;
    import java.util.concurrent.RecursiveAction;
    
    public class BatchTask extends RecursiveAction {
        
    
        // 临界值
        private static final int THRESHOLD = 10;
    
        private List<UserTestEntity> list;
    
        private UserTestMapper userTestMapper;
    
        public BatchTask(List<UserTestEntity> list, UserTestMapper userTestMapper) {
        
            this.userTestMapper = userTestMapper;
            this.list = list;
        }
    
        @Override
        protected void compute() {
        
            boolean compute = list.size() <= THRESHOLD;
            if (compute) {
        
                for (UserTestEntity userTestEntity : list) {
        
                    userTestMapper.insert(userTestEntity);
                }
            } else {
        
                List<List<UserTestEntity>> lists = BatchTask.averageAssign(list, 2);
                // 递归
                BatchTask task1 = new BatchTask(lists.get(0), userTestMapper);
                BatchTask task2 = new BatchTask(lists.get(1), userTestMapper);
                // 拆分任务，把任务压入线程队列
                invokeAll(task1, task2);
            }
        }
    
    
        /**
         * 将一组数据平均分成n组
         *
         * @param source 要分组的数据源
         * @param n      平均分成n组
         * @param <T>
         * @return
         */
        public static <T> List<List<T>> averageAssign(List<T> source, int n) {
        
            List<List<T>> result = new ArrayList<>();
            int remainder = source.size() % n;  //(先计算出余数)
            int number = source.size() / n;  //然后是商
            int offset = 0;//偏移量
            for (int i = 0; i < n; i++) {
        
                List<T> value;
                if (remainder > 0) {
        
                    value = source.subList(i * number + offset, (i + 1) * number + offset + 1);
                    remainder--;
                    offset++;
                } else {
        
                    value = source.subList(i * number + offset, (i + 1) * number + offset);
                }
                result.add(value);
            }
            return result;
        }
    
    
    }

package com.zm;
    
    
    import com.zm.entity.UserTestEntity;
    import com.zm.forkjoin.BatchTask;
    import com.zm.mapper.UserTestMapper;
    import lombok.extern.slf4j.Slf4j;
    import org.junit.jupiter.api.Test;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.boot.test.context.SpringBootTest;
    
    import java.util.ArrayList;
    import java.util.List;
    import java.util.concurrent.ExecutionException;
    import java.util.concurrent.ForkJoinPool;
    import java.util.concurrent.ForkJoinTask;
    
    @Slf4j
    @SpringBootTest
    class SpringMybatisApplicationTests {
        
    
        @Autowired
        private UserTestMapper userTestMapper;
    
        @Test
        void contextLoads() {
        
            List<UserTestEntity> userTestEntities = test1();
            log.info("=======开始========");
            ForkJoinPool pool = null;
            try {
        
                pool = new ForkJoinPool(10);
                BatchTask task = new BatchTask(userTestEntities, userTestMapper);
                ForkJoinTask<Void> submit = pool.submit(task);
                submit.get();
            } catch (InterruptedException e) {
        
                e.printStackTrace();
            } catch (ExecutionException e) {
        
                e.printStackTrace();
            } finally {
        
                // 关闭线程池
                pool.shutdown();
            }
            log.info("=======结束========");
        }
    
        /**
         * 初始化数据
         */
        private List<UserTestEntity> test1() {
        
            List<UserTestEntity> userTestEntities = new ArrayList<>();
            for (int i = 0; i < 1000000; i++) {
        
                UserTestEntity userTestEntity = new UserTestEntity();
                userTestEntity.setName1("2332131233");
                userTestEntity.setName2("3234343434");
                userTestEntity.setName3("3234343434");
                userTestEntity.setName4("3234343434");
                userTestEntity.setName5("3234343434");
                userTestEntity.setName6("3234343434");
                userTestEntity.setName7("3234343434");
                userTestEntity.setName8("3234343434");
                userTestEntity.setName9("3234343434");
                userTestEntities.add(userTestEntity);
            }
            return userTestEntities;
        }
    
    
    }

100万数据插入花费时间大概8分钟左右~

本地电脑配置 ，mysql 5.7版本  
![在这里插入图片描述][4d01a4dae74146958f8bcb5dc328b066.png]  
![在这里插入图片描述][262b969d7fa445ada375a98664ec689e.png]

[Fork_Join]: https://blog.csdn.net/qq_41604890/article/details/119271348
[4d01a4dae74146958f8bcb5dc328b066.png]: https://img-blog.csdnimg.cn/4d01a4dae74146958f8bcb5dc328b066.png
[262b969d7fa445ada375a98664ec689e.png]: https://img-blog.csdnimg.cn/262b969d7fa445ada375a98664ec689e.png