mysql diffdays_Mysql系列六：Mycat常用分片规则介绍和对应源码

淡淡的烟草味﹌ 2022-10-31 01:59 34阅读 0赞

一、Mycat常用分片规则

1. 时间类：按天分片、自然月分片、单月小时分片

2. 哈希类：Hash固定分片、日期范围Hash分片、截取数字Hash求模范围分片、截取数字Hash分片、一致性Hash分片

3. 取模类：取模分片、取模范围分片、范围求模分片

4. 其他类：枚举分片、范围约定分片、应用指定分片、冷热数据分片

下面基于源码来介绍Mycat的常用分片规则， 源码地址

二、Mycat常用分片规则介绍

说明： 分片规则都定义在rule.xml文件里面

func1

1,1,2,3,1

128,128,128,128,128

1. 自动范围分片

在rule.xml里面的配置：

rang-long

autopartition-long.txt

说明：

有3个分片，第1个分片存储的是1-500000的数据，第2个分片存储的是500001-1000000的数据，第3个分片存储的是1000001-1500000的数据

insert into employee(id, name) value(1,Tom);在第1个分片

insert into employee(id, name) value(500002,Jack);在第2个分片

insert into employee(id, name) value(1000002,Lucy);在第3个分片

对应代码：

package io.mycat.route.function;

import java.io.BufferedReader;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.util.HashSet;

import java.util.LinkedList;

import java.util.Set;

import io.mycat.config.model.rule.RuleAlgorithm;

/\*\*

\* auto partition by Long ,can be used in auto increment primary key partition

\* @author wuzhi

\*/

public class AutoPartitionByLong extends AbstractPartitionAlgorithm implements RuleAlgorithm\{

private String mapFile;

private LongRange\[\] longRongs;

private int defaultNode = -1;

@Override

public void init() \{

initialize();

public void setMapFile(String mapFile) \{

this.mapFile = mapFile;

@Override

public Integer calculate(String columnValue) \{

// columnValue = NumberParseUtil.eliminateQoute(columnValue);

try \{

long value = Long.parseLong(columnValue);

Integer rst = null;

for (LongRange longRang : this.longRongs) \{

if (value = longRang.valueStart) \{

return longRang.nodeIndx;

//数据超过范围，暂时使用配置的默认节点

if (rst == null && defaultNode >= 0) \{

return defaultNode;

return rst;

\} catch (NumberFormatException e)\{

throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please eliminate any quote and non number within it.").toString(),e);

@Override

public Integer\[\] calculateRange(String beginValue, String endValue) \{

return AbstractPartitionAlgorithm.calculateSequenceRange(this, beginValue, endValue);

@Override

public int getPartitionNum() \{

// int nPartition = longRongs.length;

/\*

\* fix \#1284 这里的统计应该统计Range的nodeIndex的distinct总数

\*/

Set distNodeIdxSet = new HashSet();

for(LongRange range : longRongs) \{

distNodeIdxSet.add(range.nodeIndx);

int nPartition = distNodeIdxSet.size();

return nPartition;

private void initialize() \{

BufferedReader in = null;

try \{

// FileInputStream fin = new FileInputStream(new File(fileMapPath));

InputStream fin = this.getClass().getClassLoader()

.getResourceAsStream(mapFile);

if (fin == null) \{

throw new RuntimeException("can't find class resource file "

\+ mapFile);

in = new BufferedReader(new InputStreamReader(fin));

LinkedList longRangeList = new LinkedList();

for (String line = null; (line = in.readLine()) != null;) \{

line = line.trim();

if (line.startsWith("\#") || line.startsWith("//")) \{

continue;

int ind = line.indexOf('=');

if (ind < 0) \{

System.out.println(" warn: bad line int " + mapFile + " :"

\+ line);

continue;

String pairs\[\] = line.substring(0, ind).trim().split("-");

long longStart = NumberParseUtil.parseLong(pairs\[0\].trim());

long longEnd = NumberParseUtil.parseLong(pairs\[1\].trim());

int nodeId = Integer.parseInt(line.substring(ind + 1)

.trim());

longRangeList

.add(new LongRange(nodeId, longStart, longEnd));

longRongs = longRangeList.toArray(new LongRange\[longRangeList

.size()\]);

\} catch (Exception e) \{

if (e instanceof RuntimeException) \{

throw (RuntimeException) e;

\} else \{

throw new RuntimeException(e);

\} finally \{

try \{

in.close();

\} catch (Exception e2) \{

public int getDefaultNode() \{

return defaultNode;

public void setDefaultNode(int defaultNode) \{

this.defaultNode = defaultNode;

static class LongRange \{

public final int nodeIndx;

public final long valueStart;

public final long valueEnd;

public LongRange(int nodeIndx, long valueStart, long valueEnd) \{

super();

this.nodeIndx = nodeIndx;

this.valueStart = valueStart;

this.valueEnd = valueEnd;

2. 枚举分片

把数据分类存储

在rule.xml里面的配置：

sharding\_id

hash-int

partition-hash-int.txt

说明： 找不到分片时设置容错规则，把数据插入到默认分片 0 里面

对应代码：

package io.mycat.route.function;

import java.io.BufferedReader;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.util.HashMap;

import java.util.HashSet;

import java.util.Map;

import java.util.Set;

import io.mycat.config.model.rule.RuleAlgorithm;

/\*\*

\* @author mycat

\*/

public class PartitionByFileMap extends AbstractPartitionAlgorithm implements RuleAlgorithm \{

private String mapFile;

private Map app2Partition;

/\*\*

\* Map app2Partition中key值的类型：默认值为0，0表示Integer，非零表示String

\*/

private int type;

/\*\*

\* 默认节点在map中的key

\*/

private static final String DEFAULT\_NODE = "DEFAULT\_NODE";

/\*\*

\* 默认节点:小于0表示不设置默认节点，大于等于0表示设置默认节点

\* 默认节点的作用：枚举分片时，如果碰到不识别的枚举值，就让它路由到默认节点

\* 如果不配置默认节点(defaultNode值小于0表示不配置默认节点)，碰到

\* 不识别的枚举值就会报错，

\* like this：can't find datanode for sharding column:column\_name val:ffffffff

\*/

private int defaultNode = -1;

@Override

public void init() \{

initialize();

public void setMapFile(String mapFile) \{

this.mapFile = mapFile;

public void setType(int type) \{

this.type = type;

public void setDefaultNode(int defaultNode) \{

this.defaultNode = defaultNode;

@Override

public Integer calculate(String columnValue) \{

try \{

Object value = columnValue;

if (type == 0) \{

value = Integer.valueOf(columnValue);

Integer rst = null;

Integer pid = app2Partition.get(value);

if (pid != null) \{

rst = pid;

\} else \{

rst = app2Partition.get(DEFAULT\_NODE);

return rst;

\} catch (NumberFormatException e)\{

throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);

@Override

public int getPartitionNum() \{

Set set = new HashSet(app2Partition.values());

int count = set.size();

return count;

private void initialize() \{

BufferedReader in = null;

try \{

// FileInputStream fin = new FileInputStream(new File(fileMapPath));

InputStream fin = this.getClass().getClassLoader()

.getResourceAsStream(mapFile);

if (fin == null) \{

throw new RuntimeException("can't find class resource file "

\+ mapFile);

in = new BufferedReader(new InputStreamReader(fin));

app2Partition = new HashMap();

for (String line = null; (line = in.readLine()) != null;) \{

line = line.trim();

if (line.startsWith("\#") || line.startsWith("//")) \{

continue;

int ind = line.indexOf('=');

if (ind = 0) \{

app2Partition.put(DEFAULT\_NODE, defaultNode);

\} catch (Exception e) \{

if (e instanceof RuntimeException) \{

throw (RuntimeException) e;

\} else \{

throw new RuntimeException(e);

\} finally \{

try \{

in.close();

\} catch (Exception e2) \{

3. Hash固定分片(固定分片Hash算法)

固定分片 Hash 算法，最多有 1024 个分片

在rule.xml里面的配置：

func1

1,1,2,3,1

128,128,128,128,128

说明：

1) partitionCount.length 必须等于 partitionLength.length

2) sum((partitionCount\[i\]\*partitionLength\[j\])) === 1024——>partitionCount\[0\]\*partitionLength\[0\]+partitionCount\[1\]\*partitionLength\[1\] === 1024

即： 1\*128+1\*128+2\*128+3\*128+1\*128 === 1024

eg：

8 个分片表

第 1 个分片表的下标为 0: 0-127

第 2 个分片表的下标为 1: 127-255

……………….

第 8 个分片表的下标为 7: 896-1024

如何确定落在哪个分片上呢？分片 id 的值与 1024 取余可确定在哪个分片上：

如 id%1024 = 128 则落在第 2 个分片上

对应代码：

package io.mycat.route.function;

import io.mycat.config.model.rule.RuleAlgorithm;

import io.mycat.route.util.PartitionUtil;

public final class PartitionByLong extends AbstractPartitionAlgorithm implements RuleAlgorithm \{

protected int\[\] count;

protected int\[\] length;

protected PartitionUtil partitionUtil;

private static int\[\] toIntArray(String string) \{

String\[\] strs = io.mycat.util.SplitUtil.split(string, ',', true);

int\[\] ints = new int\[strs.length\];

for (int i = 0; i < strs.length; ++i) \{

ints\[i\] = Integer.parseInt(strs\[i\]);

return ints;

public void setPartitionCount(String partitionCount) \{

this.count = toIntArray(partitionCount);

public void setPartitionLength(String partitionLength) \{

this.length = toIntArray(partitionLength);

@Override

public void init() \{

partitionUtil = new PartitionUtil(count, length);

@Override

public Integer calculate(String columnValue) \{

// columnValue = NumberParseUtil.eliminateQoute(columnValue);

try \{

long key = Long.parseLong(columnValue);

return partitionUtil.partition(key);

\} catch (NumberFormatException e)\{

throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please eliminate any quote and non number within it.").toString(),e);

@Override

public Integer\[\] calculateRange(String beginValue, String endValue) \{

return AbstractPartitionAlgorithm.calculateSequenceRange(this, beginValue, endValue);

// @Override

// public int getPartitionCount() \{

// int nPartition = 0;

// for(int i = 0; i < count.length; i++) \{

// nPartition += count\[i\];

// \}

// return nPartition;

// \}

4. 求模分片

分片字段 id% 分片数 = 分片下标

在rule.xml里面的配置：

mod-long

对应代码：

package io.mycat.route.function;

import java.math.BigInteger;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import io.mycat.config.model.rule.RuleAlgorithm;

/\*\*

\* number column partion by Mod operator

\* if count is 10 then 0 to 0,21 to 1 (21 % 10 =1)

\* @author wuzhih

\*/

public class PartitionByMod extends AbstractPartitionAlgorithm implements RuleAlgorithm \{

private int count;

@Override

public void init() \{

public void setCount(int count) \{

this.count = count;

@Override

public Integer calculate(String columnValue) \{

// columnValue = NumberParseUtil.eliminateQoute(columnValue);

try \{

BigInteger bigNum = new BigInteger(columnValue).abs();

return (bigNum.mod(BigInteger.valueOf(count))).intValue();

\} catch (NumberFormatException e)\{

throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please eliminate any quote and non number within it.").toString(),e);

@Override

public int getPartitionNum() \{

int nPartition = this.count;

return nPartition;

private static void hashTest() \{

PartitionByMod hash=new PartitionByMod();

hash.setCount(11);

hash.init();

int\[\] bucket=new int\[hash.count\];

Map hashed=new HashMap();

int total=1000\_0000;//数据量

int c=0;

for(int i=100\_0000;i

c++;

int h=hash.calculate(Integer.toString(i));

bucket\[h\]++;

List list=hashed.get(h);

if(list==null)\{

list=new ArrayList();

hashed.put(h, list);

list.add(i);

System.out.println(c+" "+total);

double d=0;

c=0;

int idx=0;

System.out.println("index bucket ratio");

for(int i:bucket)\{

d+=i/(double)total;

c+=i;

System.out.println(idx+++" "+i+" "+(i/(double)total));

System.out.println(d+" "+c);

System.out.println("\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*");

rehashTest(hashed.get(0));

private static void rehashTest(List partition) \{

PartitionByMod hash=new PartitionByMod();

hash.count=110;//分片数

hash.init();

int\[\] bucket=new int\[hash.count\];

int total=partition.size();//数据量

int c=0;

for(int i:partition)\{//假设分片键从100万开始

c++;

int h=hash.calculate(Integer.toString(i));

bucket\[h\]++;

System.out.println(c+" "+total);

c=0;

int idx=0;

System.out.println("index bucket ratio");

for(int i:bucket)\{

c+=i;

System.out.println(idx+++" "+i+" "+(i/(double)total));

public static void main(String\[\] args) \{

// hashTest();

PartitionByMod partitionByMod = new PartitionByMod();

partitionByMod.count=8;

partitionByMod.calculate(""6"");

partitionByMod.calculate("'6'");

5. 自然月分片

按照自然月的方式进行分片

在rule.xml里面的配置：

create\_time

partbymonth

yyyy-MM-dd

2015-01-01

说明：

如果月份超过了分片数，则通过设置 sEndDated 的值来解决

如有 3 个分片，分别插入 2015-01-12 ， 2015-02-12 、 2015-03-12 、 2016-11-12 ，月份超过了分片数，此时设置 sEndDated= 2015-04-12 表示 4 个月放一个分片，如下可知 2016-11-12 在分片 2 上

分片 0

分片 1

分片 2

1 月

5 月

9 月

2 月

6 月

10 月

3 月

7 月

11 月

4 月

8 月

12 月

对应代码：

package io.mycat.route.function;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.Collections;

import java.util.List;

import io.mycat.config.model.rule.RuleAlgorithm;

import org.apache.log4j.Logger;

/\*\*

\* 例子 按月份列分区 ，每个自然月一个分片，格式 between操作解析的范例

\* @author wzh

\*/

public class PartitionByMonth extends AbstractPartitionAlgorithm implements

RuleAlgorithm \{

private static final Logger LOGGER = Logger.getLogger(PartitionByDate.class);

private String sBeginDate;

private String dateFormat;

private String sEndDate;

private Calendar beginDate;

private Calendar endDate;

private int nPartition;

private ThreadLocal formatter;

@Override

public void init() \{

try \{

beginDate = Calendar.getInstance();

beginDate.setTime(new SimpleDateFormat(dateFormat)

.parse(sBeginDate));

formatter = new ThreadLocal() \{

@Override

protected SimpleDateFormat initialValue() \{

return new SimpleDateFormat(dateFormat);

\};

if(sEndDate!=null&&!sEndDate.equals("")) \{

endDate = Calendar.getInstance();

endDate.setTime(new SimpleDateFormat(dateFormat).parse(sEndDate));

nPartition = ((endDate.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR)) \* 12

\+ endDate.get(Calendar.MONTH) - beginDate.get(Calendar.MONTH)) + 1;

if (nPartition <= 0) \{

throw new java.lang.IllegalArgumentException("Incorrect time range for month partitioning!");

\} else \{

nPartition = -1;

\} catch (ParseException e) \{

throw new java.lang.IllegalArgumentException(e);

/\*\*

\* For circulatory partition, calculated value of target partition needs to be

\* rotated to fit the partition range

\*/

private int reCalculatePartition(int targetPartition) \{

/\*\*

\* If target date is previous of start time of partition setting, shift

\* the delta range between target and start date to be positive value

\*/

if (targetPartition = nPartition) \{

targetPartition = targetPartition % nPartition;

return targetPartition;

@Override

public Integer calculate(String columnValue) \{

try \{

int targetPartition;

Calendar curTime = Calendar.getInstance();

curTime.setTime(formatter.get().parse(columnValue));

targetPartition = ((curTime.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR))

\* 12 + curTime.get(Calendar.MONTH)

\- beginDate.get(Calendar.MONTH));

/\*\*

\* For circulatory partition, calculated value of target partition needs to be

\* rotated to fit the partition range

\*/

if (nPartition > 0) \{

targetPartition = reCalculatePartition(targetPartition);

return targetPartition;

\} catch (ParseException e) \{

throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);

@Override

public Integer\[\] calculateRange(String beginValue, String endValue) \{

try \{

int startPartition, endPartition;

Calendar partitionTime = Calendar.getInstance();

SimpleDateFormat format = new SimpleDateFormat(dateFormat);

partitionTime.setTime(format.parse(beginValue));

startPartition = ((partitionTime.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR))

\* 12 + partitionTime.get(Calendar.MONTH)

\- beginDate.get(Calendar.MONTH));

partitionTime.setTime(format.parse(endValue));

endPartition = ((partitionTime.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR))

\* 12 + partitionTime.get(Calendar.MONTH)

\- beginDate.get(Calendar.MONTH));

List list = new ArrayList();

while (startPartition <= endPartition) \{

Integer nodeValue = reCalculatePartition(startPartition);

if (Collections.frequency(list, nodeValue) < 1)

list.add(nodeValue);

startPartition++;

int size = list.size();

return (list.toArray(new Integer\[size\]));

\} catch (ParseException e) \{

LOGGER.error("error",e);

return new Integer\[0\];

@Override

public int getPartitionNum() \{

int nPartition = this.nPartition;

return nPartition;

public void setsBeginDate(String sBeginDate) \{

this.sBeginDate = sBeginDate;

public void setDateFormat(String dateFormat) \{

this.dateFormat = dateFormat;

public void setsEndDate(String sEndDate) \{

this.sEndDate = sEndDate;

6. 匹配求模分片

根据 prefixLength 截取 n 个字符 并 charAt(i) 每个字符的值进行累加得到一个整数 ，然后和分区长度 patternValue 进行求模，得出的值就是分区编号。

在rule.xml里面的配置：

partitionbyprefixpattern

说明：

有下面这种类型的数据

年月 + 大区 + 流水编号

201801 01 10001

就可以采用匹配求模分片，把分片字段 columns 取前 6 个字符串 201801 并 charAt(i) 每个字符的值进行累加得到一个整数，然后和分区长度 3 进行求模，得出的值就是分区编号

对应代码：

package io.mycat.route.function;

import java.io.BufferedReader;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.util.HashSet;

import java.util.LinkedList;

import java.util.Set;

import io.mycat.config.model.rule.RuleAlgorithm;

import io.mycat.route.function.AutoPartitionByLong.LongRange;

/\*\*

\* partition by Prefix length ,can be used in String partition

\* @author hexiaobin

\*/

public class PartitionByPrefixPattern extends AbstractPartitionAlgorithm implements RuleAlgorithm \{

private static final int PARTITION\_LENGTH = 1024;

private int patternValue = PARTITION\_LENGTH;// 分区长度，取模数值(默认为1024)

private int prefixLength;// 字符前几位进行ASCII码取和

private String mapFile;

private LongRange\[\] longRongs;

@Override

public void init() \{

initialize();

public void setMapFile(String mapFile) \{

this.mapFile = mapFile;

public void setPatternValue(int patternValue) \{

this.patternValue = patternValue;

public void setPrefixLength(int prefixLength) \{

this.prefixLength = prefixLength;

@Override

public Integer calculate(String columnValue) \{

try \{

int Length = Integer.valueOf(prefixLength);

Length = columnValue.length() < Length ? columnValue.length() : Length;

int sum = 0;

for (int i = 0; i < Length; i++) \{

sum = sum + columnValue.charAt(i);

Integer rst = null;

for (LongRange longRang : this.longRongs) \{

long hash = sum % patternValue;

if (hash = longRang.valueStart) \{

return longRang.nodeIndx;

return rst;

\} catch (NumberFormatException e)\{

throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please eliminate any quote and non number within it.").toString(),e);

@Override

public int getPartitionNum() \{

// int nPartition = this.longRongs.length;

/\*

\* fix \#1284 这里的统计应该统计Range的nodeIndex的distinct总数

\*/

Set distNodeIdxSet = new HashSet();

for(LongRange range : longRongs) \{

distNodeIdxSet.add(range.nodeIndx);

int nPartition = distNodeIdxSet.size();

return nPartition;

private void initialize() \{

BufferedReader in = null;

try \{

// FileInputStream fin = new FileInputStream(new File(fileMapPath));

InputStream fin = this.getClass().getClassLoader()

.getResourceAsStream(mapFile);

if (fin == null) \{

throw new RuntimeException("can't find class resource file "

\+ mapFile);

in = new BufferedReader(new InputStreamReader(fin));

LinkedList longRangeList = new LinkedList();

for (String line = null; (line = in.readLine()) != null;) \{

line = line.trim();

if (line.startsWith("\#") || line.startsWith("//")) \{

continue;

int ind = line.indexOf('=');

if (ind < 0) \{

System.out.println(" warn: bad line int " + mapFile + " :"

\+ line);

continue;

String pairs\[\] = line.substring(0, ind).trim().split("-");

long longStart = NumberParseUtil.parseLong(pairs\[0\].trim());

long longEnd = NumberParseUtil.parseLong(pairs\[1\].trim());

int nodeId = Integer.parseInt(line.substring(ind + 1)

.trim());

longRangeList

.add(new LongRange(nodeId, longStart, longEnd));

longRongs = longRangeList.toArray(new LongRange\[longRangeList

.size()\]);

\} catch (Exception e) \{

if (e instanceof RuntimeException) \{

throw (RuntimeException) e;

\} else \{

throw new RuntimeException(e);

\} finally \{

try \{

in.close();

\} catch (Exception e2) \{

static class LongRange \{

public final int nodeIndx;

public final long valueStart;

public final long valueEnd;

public LongRange(int nodeIndx, long valueStart, long valueEnd) \{

super();

this.nodeIndx = nodeIndx;

this.valueStart = valueStart;

this.valueEnd = valueEnd;

7. 冷热数据分片

根据日期查询日志数据冷热数据分布 ，最近 n 个月的到实时交易库查询，超过 n 个月的按照 m 天分片 。

在rule.xml里面的配置：

create\_time

sharding-by-hotdate

yyyy-MM-dd

对应代码：

package io.mycat.route.function;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Calendar;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import io.mycat.config.model.rule.RuleAlgorithm;

/\*\*

\* 根据日期查询日志数据 冷热数据分布 ，最近n个月的到实时交易库查询，超过n个月的按照m天分片

\* @author sw

create\_time

sharding-by-hotdate

yyyy-MM-dd

\*/

public class PartitionByHotDate extends AbstractPartitionAlgorithm implements RuleAlgorithm \{

private static final Logger LOGGER = LoggerFactory.getLogger(PartitionByHotDate.class);

private String dateFormat;

private String sLastDay;

private String sPartionDay;

private long sLastTime;

private long partionTime;

private ThreadLocal formatter;

private long beginDate;

private static final long oneDay = 86400000;

@Override

public void init() \{

try \{

formatter = new ThreadLocal() \{

@Override

protected SimpleDateFormat initialValue() \{

return new SimpleDateFormat(dateFormat);

\};

sLastTime = Integer.valueOf(sLastDay);

partionTime = Integer.parseInt(sPartionDay) \* oneDay;

\} catch (Exception e) \{

throw new java.lang.IllegalArgumentException(e);

@Override

public Integer calculate(String columnValue) \{

Integer targetPartition = -1;

try \{

long targetTime = formatter.get().parse(columnValue).getTime();

Calendar now = Calendar.getInstance();

long nowTime = now.getTimeInMillis();

beginDate = nowTime - sLastTime \* oneDay;

long diffDays = (nowTime - targetTime) / (1000 \* 60 \* 60 \* 24) + 1;

if(diffDays-sLastTime <= 0 || diffDays<0 )\{

targetPartition = 0;

\}else\{

targetPartition = (int) ((beginDate - targetTime) / partionTime) + 1;

LOGGER.debug("PartitionByHotDate calculate for " + columnValue + " return " + targetPartition);

return targetPartition;

\} catch (ParseException e) \{

throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);

@Override

public Integer\[\] calculateRange(String beginValue, String endValue) \{

Integer\[\] targetPartition = null;

try \{

long startTime = formatter.get().parse(beginValue).getTime();

long endTime = formatter.get().parse(endValue).getTime();

Calendar now = Calendar.getInstance();

long nowTime = now.getTimeInMillis();

long limitDate = nowTime - sLastTime \* oneDay;

long diffDays = (nowTime - startTime) / (1000 \* 60 \* 60 \* 24) + 1;

if(diffDays-sLastTime <= 0 || diffDays 0)\{

endTime = limitDate;

hasLimit = true;

begin = this.calculate(formatter.get().format(endTime));

if(begin == null || end == null)\{

return re;

if (end >= begin) \{

int len = end-begin+1;

if(hasLimit)\{

re = new Integer\[len+1\];

re\[0\] = 0;

for(int i =0;i

re\[i+1\]=begin+i;

\}else\{

re = new Integer\[len\];

for(int i=0;i

re\[i\]=begin+i;

return re;

\}else\{

return re;

\} catch (ParseException e) \{

throw new IllegalArgumentException(new StringBuilder().append("endValue:").append(endValue).append(" Please check if the format satisfied.").toString(),e);

return targetPartition;

public void setsPartionDay(String sPartionDay) \{

this.sPartionDay = sPartionDay;

public void setDateFormat(String dateFormat) \{

this.dateFormat = dateFormat;

public String getsLastDay() \{

return sLastDay;

public void setsLastDay(String sLastDay) \{

this.sLastDay = sLastDay;

8. 一致性哈希分片

1)首先求出mysql服务器(节点)的哈希值，并将其配置到0～2^32的圆(continuum)上。

2)为每台mysql服务器物理节点虚拟出多个虚拟节点，并计算hash值映射到相同的圆上。

3)然后从数据映射到的mysql服务器虚拟节点的位置开始顺时针查找，将数据保存到找到的第一个mysql服务器上。如果超过2 32 仍然找不到服务器，就会保存到第一台mysql服务器上。

![4028616b683fabd02974ba00bec57b12.png][]

特点：解决数据均匀分布

在rule.xml里面的配置：

murmur

160

对应代码：

package io.mycat.route.function;

import java.io.BufferedReader;

import java.io.ByteArrayInputStream;

import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.OutputStream;

import java.nio.charset.Charset;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Properties;

import java.util.SortedMap;

import java.util.TreeMap;

import com.google.common.hash.HashFunction;

import com.google.common.hash.Hashing;

import io.mycat.config.model.rule.RuleAlgorithm;

import io.mycat.util.exception.MurmurHashException;

/\*\*

\* consistancy hash, murmur hash

\* implemented by Guava

\* @author wuzhih

\*/

public class PartitionByMurmurHash extends AbstractPartitionAlgorithm implements RuleAlgorithm \{

private static final int DEFAULT\_VIRTUAL\_BUCKET\_TIMES=160;

private static final int DEFAULT\_WEIGHT=1;

private static final Charset DEFAULT\_CHARSET=Charset.forName("UTF-8");

private int seed;

private int count;

private int virtualBucketTimes=DEFAULT\_VIRTUAL\_BUCKET\_TIMES;

private Map weightMap=new HashMap();

// private String bucketMapPath;

private HashFunction hash;

private SortedMap bucketMap;

@Override

public void init() \{

try\{

bucketMap=new TreeMap();

// boolean serializableBucketMap=bucketMapPath!=null && bucketMapPath.length()>0;

// if(serializableBucketMap)\{

// File bucketMapFile=new File(bucketMapPath);

// if(bucketMapFile.exists() && bucketMapFile.length()>0)\{

// loadBucketMapFile();

// return;

// \}

generateBucketMap();

// if(serializableBucketMap)\{

// storeBucketMap();

// \}

\}catch(Exception e)\{

throw new MurmurHashException(e);

private void generateBucketMap()\{

hash=Hashing.murmur3\_32(seed);//计算一致性哈希的对象

for(int i=0;i

StringBuilder hashName=new StringBuilder("SHARD-").append(i);

for(int n=0,shard=virtualBucketTimes\*getWeight(i);n0?weight:1);

// /\*\*

// \* 保存一致性hash的虚拟节点文件路径。

// \* 如果这个文件不存在或是空文件就按照指定的count, weightMapFile等构造新的MurmurHash数据结构并保存到这个路径的文件里。

// \* 如果这个文件已存在且不是空文件就加载这个文件里的内容作为MurmurHash数据结构，此时其它参数都忽略。

// \* 除第一次以外在之后增加节点时可以直接修改这个文件，不过不推荐这么做。如果节点数量变化了，推荐删除这个文件。

// \* 可以不指定这个路径，不指定路径时不会保存murmur hash

// \* @param bucketMapPath

// \*/

// public void setBucketMapPath(String bucketMapPath)\{

// this.bucketMapPath=bucketMapPath;

// \}

@Override

public Integer calculate(String columnValue) \{

SortedMap tail = bucketMap.tailMap(hash.hashUnencodedChars(columnValue).asInt());

if (tail.isEmpty()) \{

return bucketMap.get(bucketMap.firstKey());

return tail.get(tail.firstKey());

@Override

public int getPartitionNum() \{

int nPartition = this.count;

return nPartition;

private static void hashTest() throws IOException\{

PartitionByMurmurHash hash=new PartitionByMurmurHash();

hash.count=10;//分片数

hash.init();

int\[\] bucket=new int\[hash.count\];

Map hashed=new HashMap();

int total=1000\_0000;//数据量

int c=0;

for(int i=100\_0000;i

c++;

int h=hash.calculate(Integer.toString(i));

bucket\[h\]++;

List list=hashed.get(h);

if(list==null)\{

list=new ArrayList();

hashed.put(h, list);

list.add(i);

System.out.println(c+" "+total);

double d=0;

c=0;

int idx=0;

System.out.println("index bucket ratio");

for(int i:bucket)\{

d+=i/(double)total;

c+=i;

System.out.println(idx+++" "+i+" "+(i/(double)total));

System.out.println(d+" "+c);

Properties props=new Properties();

for(Map.Entry entry:hash.bucketMap.entrySet())\{

props.setProperty(entry.getKey().toString(), entry.getValue().toString());

ByteArrayOutputStream out=new ByteArrayOutputStream();

props.store(out, null);

props.clear();

props.load(new ByteArrayInputStream(out.toByteArray()));

System.out.println(props);

System.out.println("\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*");

// rehashTest(hashed.get(0));

private static void rehashTest(List partition)\{

PartitionByMurmurHash hash=new PartitionByMurmurHash();

hash.count=12;//分片数

hash.init();

int\[\] bucket=new int\[hash.count\];

int total=partition.size();//数据量

int c=0;

for(int i:partition)\{//假设分片键从100万开始

c++;

int h=hash.calculate(Integer.toString(i));

bucket\[h\]++;

System.out.println(c+" "+total);

c=0;

int idx=0;

System.out.println("index bucket ratio");

for(int i:bucket)\{

c+=i;

System.out.println(idx+++" "+i+" "+(i/(double)total));

public static void main(String\[\] args) throws IOException \{

hashTest();

注意：本文来源网络/媒体，本站无法对本文内容的真实性、完整性、及时性、原创性提供任何保证，

请您自行验证核实并承担相关的风险与后果！

CoLaBug.com遵循\[CC BY-SA 4.0\]分享并保持客观立场，本站不承担此类作品侵权行为的直接责任及连带责任。

如您有版权、意见、投诉等问题，请通过\[eMail\]联系我们处理，如需商业授权请联系原作者/原网站。

[4028616b683fabd02974ba00bec57b12.png]: https://img.colabug.com/2018/08/4028616b683fabd02974ba00bec57b12.png

mysql diffdays_Mysql系列六：Mycat常用分片规则介绍和对应源码

发表评论取消回复

还没有评论，来说两句吧...

相关阅读

相关 MySQL - 分库分表、MyCat配置、分片规则

相关 Mycat【Mycat分片规则（取模、分片枚举、范围约定）】(六)-全面详解（学习总结---从入门到深化）

相关 mycat 常用分片规则使用详解

相关 Mycat--分片规则

相关 mysql diffdays_Mysql系列六：Mycat常用分片规则介绍和对应源码

相关 mycat分片规则详解+实例演示

相关 mycat分布式mysql中间件（分片规则）

相关 Mycat学习实战-Mycat分片

相关 mycat 分片规则

相关 Mycat——常用的分片规则