From d295ef53288375af40c696fb669317e87ad3e008 Mon Sep 17 00:00:00 2001 From: zhangbingbing Date: Fri, 16 Oct 2020 12:56:16 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9F=BA=E4=BA=8E?= =?UTF-8?q?=E9=9B=AA=E8=8A=B1=E7=AE=97=E6=B3=95=E7=9A=84IdGenerator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../SystemClockCallbackException.java | 19 +++ .../generator/impl/SnowflakeIdGenerator.java | 160 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/exception/SystemClockCallbackException.java create mode 100644 tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGenerator.java diff --git a/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/exception/SystemClockCallbackException.java b/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/exception/SystemClockCallbackException.java new file mode 100644 index 0000000..acbe638 --- /dev/null +++ b/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/exception/SystemClockCallbackException.java @@ -0,0 +1,19 @@ +package com.xiaoju.uemc.tinyid.base.exception; + +/** + * @author zhangbingbing + * @Description 系统时间回调异常 + * @date 2020/10/16 + */ +public class SystemClockCallbackException extends RuntimeException { + + private static final long serialVersionUID = -6264588182225994225L; + + public SystemClockCallbackException(String msg) { + super(msg); + } + + public SystemClockCallbackException(String msg, Throwable cause) { + super(msg, cause); + } +} diff --git a/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGenerator.java b/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGenerator.java new file mode 100644 index 0000000..9bc07f6 --- /dev/null +++ b/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGenerator.java @@ -0,0 +1,160 @@ +package com.xiaoju.uemc.tinyid.base.generator.impl; + +import com.xiaoju.uemc.tinyid.base.exception.SystemClockCallbackException; +import com.xiaoju.uemc.tinyid.base.generator.IdGenerator; + +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * 雪花算法实现：0(固定位)+41(当前时间戳-startAt)+10(机器号)+12(序号) + * + * @author zhangbingbing + * @date 2020/10/16 + */ +public class SnowflakeIdGenerator implements IdGenerator { + + private static final Logger logger = Logger.getLogger(SnowflakeIdGenerator.class.getName()); + /** + * 2020-10-15 00:00:00 (GMT+8) 毫秒 + */ + private final long startAt = 1602691200000L; + private final int workIdBits = 10; + private final int maxWorkId = ~(-1 << workIdBits); + private final int sequenceBits = 12; + private final int workIdShift = sequenceBits; + private final int timestampShift = workIdBits + sequenceBits; + private final int sequenceMask = ~(-1 << sequenceBits); + private final Random random = new Random(); + private final int workId; + private int sequence = 0; + private long lastTimestamp = -1L; + + public SnowflakeIdGenerator(int workId) { + if (workId > maxWorkId) { + throw new IllegalStateException("the work id " + workId + " greater than max work Id " + maxWorkId); + } + this.workId = workId; + SnowflakeIdGenerator.logger.info("snowflake work id " + workId); + } + + @Override + public Long nextId() { + return null; + } + + @Override + public List nextId(Integer batchSize) { + return null; + } + + + private synchronized long doNextId() { + long now = now(); + // 时钟回调了 + if (now < lastTimestamp) { + long offset = lastTimestamp - now; + if (offset > 5) { + throw new SystemClockCallbackException("system clock callback slow " + offset); + } + try { + this.wait(offset << 1); + } catch (InterruptedException e) { + throw new SystemClockCallbackException("system clock callback slow " + offset, e); + } + } + if (now == lastTimestamp) { + sequence = (sequence + 1) & sequenceMask; + // 该毫秒内的sequence已经用完了 + if (sequence == 0) { + sequence = random.nextInt(100); + now = tillNextMill(lastTimestamp); + } + } + // 从新的毫秒开始 + if (now > lastTimestamp) { + sequence = random.nextInt(100); + } + lastTimestamp = now; + return toId(lastTimestamp, workId, sequence); + } + + private synchronized Set doNextIds(int batchSize) { + if ((batchSize & sequenceMask) == 0) { + throw new IllegalArgumentException("batch size " + batchSize); + } + long now = now(); + if (now < lastTimestamp) { + long offset = lastTimestamp - now; + if (offset > 5) { + throw new SystemClockCallbackException("system clock callback slow " + offset); + } + try { + this.wait(offset << 1); + } catch (InterruptedException e) { + throw new SystemClockCallbackException("system clock callback slow " + offset, e); + } + } + Set nextIds = new HashSet<>(batchSize); + while (nextIds.size() < batchSize) { + // 在本毫秒 + if (now == lastTimestamp) { + sequence = (sequence + 1) & sequenceMask; + // 本毫秒内的sequence用完了 + if (sequence == 0) { + sequence = random.nextInt(100); + now = tillNextMill(lastTimestamp); + } + nextIds.add(toId(now, workId, sequence)); + continue; + } + // 在新的毫秒 + if (now > lastTimestamp) { + sequence = random.nextInt(100); + int loop = batchSize - nextIds.size(); + for (int i = 0; i < loop; i++) { + sequence = sequence + 1; + nextIds.add(toId(now, workId, sequence)); + } + } + } + lastTimestamp = now; + return nextIds; + } + + private long toId(long timestamp, int workId, int sequence) { + return ((timestamp - startAt) << timestampShift) | (workId << workIdShift) | sequence; + } + + /** + * 等待下个毫秒，防止等待期间系统时钟被回调，导致方法一直轮询 + */ + private long tillNextMill(long lastTimestamp) { + long timestamp; + long offset; + while (true) { + timestamp = now(); + offset = lastTimestamp - timestamp; + if (offset < 0) { + return timestamp; + } + if (offset >= 5) { // 系统时钟回调时间大于5ms + throw new SystemClockCallbackException("timestamp check error,last timestamp " + lastTimestamp + ",now " + timestamp); + } + if (offset >= 2) { // 系统时钟回调时间大于等于2ms + try { + TimeUnit.MILLISECONDS.sleep(offset); + } catch (InterruptedException ignore) { + } + } + } + } + + private long now() { + return System.currentTimeMillis(); + } +} From a51a5d4c61c17997511156c6450dae0a8bd83e85 Mon Sep 17 00:00:00 2001 From: zhangbingbing Date: Fri, 16 Oct 2020 13:36:59 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E4=BE=8B=EF=BC=9ASnowflakeIdGeneratorTest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../generator/impl/SnowflakeIdGenerator.java | 11 ++-- .../impl/SnowflakeIdGeneratorTest.java | 61 +++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 tinyid-client/src/test/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGeneratorTest.java diff --git a/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGenerator.java b/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGenerator.java index 9bc07f6..b872b13 100644 --- a/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGenerator.java +++ b/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGenerator.java @@ -3,10 +3,9 @@ import com.xiaoju.uemc.tinyid.base.exception.SystemClockCallbackException; import com.xiaoju.uemc.tinyid.base.generator.IdGenerator; -import java.util.HashSet; +import java.util.ArrayList; import java.util.List; import java.util.Random; -import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -44,12 +43,12 @@ public SnowflakeIdGenerator(int workId) { @Override public Long nextId() { - return null; + return doNextId(); } @Override public List nextId(Integer batchSize) { - return null; + return doNextIds(batchSize); } @@ -83,7 +82,7 @@ private synchronized long doNextId() { return toId(lastTimestamp, workId, sequence); } - private synchronized Set doNextIds(int batchSize) { + private synchronized List doNextIds(int batchSize) { if ((batchSize & sequenceMask) == 0) { throw new IllegalArgumentException("batch size " + batchSize); } @@ -99,7 +98,7 @@ private synchronized Set doNextIds(int batchSize) { throw new SystemClockCallbackException("system clock callback slow " + offset, e); } } - Set nextIds = new HashSet<>(batchSize); + List nextIds = new ArrayList<>(batchSize); while (nextIds.size() < batchSize) { // 在本毫秒 if (now == lastTimestamp) { diff --git a/tinyid-client/src/test/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGeneratorTest.java b/tinyid-client/src/test/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGeneratorTest.java new file mode 100644 index 0000000..fb731e9 --- /dev/null +++ b/tinyid-client/src/test/java/com/xiaoju/uemc/tinyid/base/generator/impl/SnowflakeIdGeneratorTest.java @@ -0,0 +1,61 @@ +package com.xiaoju.uemc.tinyid.base.generator.impl; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; + +/** + * @author zhangbingbing + * @date 2020/10/16 + * @see SnowflakeIdGenerator 测试用例 + */ +public class SnowflakeIdGeneratorTest { + + private final SnowflakeIdGenerator idGenerator = new SnowflakeIdGenerator(1); + private final SnowflakeIdGenerator idGenerator2 = new SnowflakeIdGenerator(2); + + @Test + public void testNextId() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(2); + final int size = 5000; + final Set idSet = new HashSet<>(size); + final Set idSet2 = new HashSet<>(size); + new Thread(new Runnable() { + @Override + public void run() { + for (int i = 0; i < size; i++) { + idSet.add(idGenerator.nextId()); + } + latch.countDown(); + } + }).start(); + new Thread(new Runnable() { + @Override + public void run() { + for (int i = 0; i < size; i++) { + idSet2.add(idGenerator2.nextId()); + } + latch.countDown(); + } + }).start(); + latch.await(); + Assert.assertEquals(size, idSet.size()); + Assert.assertEquals(size, idSet2.size()); + idSet.removeAll(idSet2); + // idSet idSet2没有重复的id + Assert.assertEquals(size, idSet.size()); + } + + @Test + public void testNextIds() { + Set idSet = new HashSet<>(5000); + for (int i = 0; i < 5; i++) { + idSet.addAll(idGenerator.nextId(1000)); + } + Assert.assertEquals(5000, idSet.size()); + } + +} \ No newline at end of file From e147fe6f6a55a63d027d0e6f3e96af7169b51b06 Mon Sep 17 00:00:00 2001 From: zhangbingbing Date: Fri, 16 Oct 2020 15:35:47 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E9=9B=AA=E8=8A=B1=E7=AE=97=E6=B3=95=E4=B8=ADworkerId=E7=9A=84s?= =?UTF-8?q?ervice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaoju/uemc/tinyid/base/util/IPUtil.java | 52 +++ tinyid-server/pom.xml | 6 + .../server/common/annotation/Module.java | 24 ++ .../common/annotation/ModuleCondition.java | 39 +++ .../config/SnowflakeWorkerIdConfig.java | 40 +++ .../service/SnowflakeWorkIdService.java | 13 + .../impl/SnowflakeWorkerIdServiceImpl.java | 303 ++++++++++++++++++ .../resources/offline/application.properties | 9 +- .../resources/online/application.properties | 7 +- .../SnowflakeWorkerIdServiceImplTest.java | 45 +++ 10 files changed, 534 insertions(+), 4 deletions(-) create mode 100644 tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/util/IPUtil.java create mode 100644 tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/common/annotation/Module.java create mode 100644 tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/common/annotation/ModuleCondition.java create mode 100644 tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/config/SnowflakeWorkerIdConfig.java create mode 100644 tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/SnowflakeWorkIdService.java create mode 100644 tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeWorkerIdServiceImpl.java create mode 100644 tinyid-server/src/test/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeWorkerIdServiceImplTest.java diff --git a/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/util/IPUtil.java b/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/util/IPUtil.java new file mode 100644 index 0000000..db142fb --- /dev/null +++ b/tinyid-base/src/main/java/com/xiaoju/uemc/tinyid/base/util/IPUtil.java @@ -0,0 +1,52 @@ +package com.xiaoju.uemc.tinyid.base.util; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; + +/** + * @author zhangbingbing + * @date 2020/10/16 + */ +public enum IPUtil { + ; + + public static String getHostAddress() throws SocketException { + return IPUtil.getHostAddress(null).get(0); + } + + /** + * 获取已激活网卡的ip地址 + * + * @param interfaceName 网卡地址，null则获取所有 + * @return List + */ + public static List getHostAddress(String interfaceName) throws SocketException { + List ips = new ArrayList<>(5); + Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces(); + while (networkInterfaces.hasMoreElements()) { + NetworkInterface networkInterface = networkInterfaces.nextElement(); + Enumeration allAddress = networkInterface.getInetAddresses(); + while (allAddress.hasMoreElements()) { + InetAddress address = allAddress.nextElement(); + if (address.isLoopbackAddress()) { + continue; + } + if (address instanceof Inet6Address) { + continue; + } + String hostAddress = address.getHostAddress(); + if (null == interfaceName) { + ips.add(hostAddress); + } else if (interfaceName.equals(networkInterface.getDisplayName())) { + ips.add(hostAddress); + } + } + } + return ips; + } +} diff --git a/tinyid-server/pom.xml b/tinyid-server/pom.xml index 2849537..5be4f03 100755 --- a/tinyid-server/pom.xml +++ b/tinyid-server/pom.xml @@ -13,6 +13,7 @@ 1.5.9.RELEASE + 2.6.0 @@ -69,6 +70,11 @@ commons-beanutils 1.9.3 + + org.apache.curator + curator-recipes + ${curator.version} + diff --git a/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/common/annotation/Module.java b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/common/annotation/Module.java new file mode 100644 index 0000000..2aa8d3f --- /dev/null +++ b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/common/annotation/Module.java @@ -0,0 +1,24 @@ +package com.xiaoju.uemc.tinyid.server.common.annotation; + +import org.springframework.context.annotation.Conditional; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Conditional(ModuleCondition.class) +public @interface Module { + + /** + * 前缀 + */ + String prefix() default ""; + + /** + * 匹配的值 + */ + String[] value(); +} diff --git a/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/common/annotation/ModuleCondition.java b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/common/annotation/ModuleCondition.java new file mode 100644 index 0000000..3ae15df --- /dev/null +++ b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/common/annotation/ModuleCondition.java @@ -0,0 +1,39 @@ +package com.xiaoju.uemc.tinyid.server.common.annotation; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; +import org.springframework.util.MultiValueMap; + +public class ModuleCondition implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + final MultiValueMap attributes = metadata.getAllAnnotationAttributes(Module.class.getName()); + if (attributes == null) { + return true; + } + String prefix = ""; + Object prefixValue = attributes.getFirst("prefix"); + if (prefixValue != null) { + prefix = prefixValue.toString(); + } + final Environment environment = context.getEnvironment(); + for (Object value : attributes.get("value")) { + String[] moduleName = (String[]) value; + for (String module : moduleName) { + String propertyName; + if (prefix.equals("")) { + propertyName = module; + } else { + propertyName = prefix + "." + module; + } + if (environment.getProperty(propertyName, boolean.class, false)) { + return true; + } + } + } + return false; + } +} diff --git a/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/config/SnowflakeWorkerIdConfig.java b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/config/SnowflakeWorkerIdConfig.java new file mode 100644 index 0000000..8203e43 --- /dev/null +++ b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/config/SnowflakeWorkerIdConfig.java @@ -0,0 +1,40 @@ +package com.xiaoju.uemc.tinyid.server.config; + +import com.xiaoju.uemc.tinyid.server.common.annotation.Module; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author zhangbingbing + * @date 2020/10/16 + */ +@Module(value = "snowflake.enable") +@Configuration +public class SnowflakeWorkerIdConfig { + + /** + * 定时上传数据到zk的定时任务线程池 + */ + @Bean + @Module(value = "snowflake.enable") + public ScheduledExecutorService updateDataToZKScheduledExecutorService() { + final AtomicInteger threadIncr = new AtomicInteger(0); + return new ScheduledThreadPoolExecutor(1, new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + int incr = threadIncr.incrementAndGet(); + if (incr >= 1000) { + threadIncr.set(0); + incr = 1; + } + return new Thread(r, "upload-data-to-zk-schedule-thread" + incr); + } + }, new ThreadPoolExecutor.CallerRunsPolicy()); + } +} diff --git a/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/SnowflakeWorkIdService.java b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/SnowflakeWorkIdService.java new file mode 100644 index 0000000..78160b5 --- /dev/null +++ b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/SnowflakeWorkIdService.java @@ -0,0 +1,13 @@ +package com.xiaoju.uemc.tinyid.server.service; + +/** + * @author zhangbingbing + * @date 2020/10/16 + */ +public interface SnowflakeWorkIdService { + + /** + * 获取雪花算法中的workId + */ + int workId(); +} diff --git a/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeWorkerIdServiceImpl.java b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeWorkerIdServiceImpl.java new file mode 100644 index 0000000..11438f3 --- /dev/null +++ b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeWorkerIdServiceImpl.java @@ -0,0 +1,303 @@ +package com.xiaoju.uemc.tinyid.server.service.impl; + +import com.xiaoju.uemc.tinyid.base.exception.SystemClockCallbackException; +import com.xiaoju.uemc.tinyid.base.util.IPUtil; +import com.xiaoju.uemc.tinyid.server.common.annotation.Module; +import com.xiaoju.uemc.tinyid.server.service.SnowflakeWorkIdService; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryUntilElapsed; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.ResourceUtils; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * 通过zk的有序节点，来获取一个自增的数作为workerId + * + * @author zhangbingbing + * @date 2020/10/16 + */ +@Service +@Module(value = "snowflake.enable") +public class SnowflakeWorkerIdServiceImpl implements SnowflakeWorkIdService { + + private static final Logger log = LoggerFactory.getLogger(SnowflakeWorkerIdServiceImpl.class); + + @Autowired + @Qualifier("updateDataToZKScheduledExecutorService") + private ScheduledExecutorService scheduledExecutorService; + /** + * 保存所有数据持久的节点 + */ + private static final String ZK_PATH = "/tinyid/snowflake/forever"; + /** + * 持久化workerId，文件存放位置 + */ + private static final String DUMP_PATH = "workerID/workerID.properties"; + /** + * 是否在连接不上zk时，使用之前备份的workerId + */ + @Value("${snowflake.zk.load-worker-id-from-file-when-zk-down:true}") + private boolean loadWorkerIdFromFileWhenZkDown; + /** + * zk连接信息 + */ + @Value("${snowflake.zk.connection-string}") + private String connectionString; + /** + * 本机地址 + */ + private String ip; + /** + * 本机端口 + */ + @Value("${server.port}") + private Integer port; + /** + * 上次更新数据时间 + */ + private long lastUpdateAt; + /** + * workerId + */ + private volatile Integer workerId; + + @Override + public int workId() { + try { + if (workerId != null) { + return workerId; + } + doInitWorkId(); + return workerId; + } catch (Exception e) { + throw new IllegalStateException("获取workerId失败", e); + } + } + + private synchronized void doInitWorkId() throws Exception { + if (workerId != null) { + return; + } + try { + ip = IPUtil.getHostAddress(); + String localZKPath = ZK_PATH + "/" + ip + ":" + port; + CuratorFramework client = getZKConnection(); + final Stat stat = client.checkExists().forPath(ZK_PATH); + // 不存在根结点，第一次使用，创建根结点 + if (stat == null) { + // 创建有序永久结点 /tinyid/snowflake/forever/ip:port-xxx,并上传数据 + localZKPath = createPersistentSequentialNode(client, localZKPath, buildData()); + workerId = getWorkerId(localZKPath); + // 持久化workerId + updateWorkerId(workerId); + // 定时上报本机时间到zk + scheduledUploadTimeToZK(client, localZKPath); + return; + } + // Map + Map localAddressWorkerIdMap = new HashMap<>(16); + // Map + Map localAddressPathMap = new HashMap<>(16); + for (String key : client.getChildren().forPath(ZK_PATH)) { + final String[] split = key.split("-"); + localAddressPathMap.put(split[0], key); + // value=zk有序结点的需要 + localAddressWorkerIdMap.put(split[0], Integer.valueOf(split[1])); + } + String localAddress = ip + ":" + port; + workerId = localAddressWorkerIdMap.get(localAddress); + if (workerId != null) { + localZKPath = ZK_PATH + "/" + localAddressPathMap.get(localAddress); + // 校验时间是否回调 + checkTimestamp(client, localZKPath); + scheduledUploadTimeToZK(client, localZKPath); + updateWorkerId(workerId); + return; + } + localZKPath = createPersistentSequentialNode(client, localZKPath, buildData()); + workerId = Integer.parseInt((localZKPath.split("-"))[1]); + scheduledUploadTimeToZK(client, localZKPath); + updateWorkerId(workerId); + } catch (Exception e) { + if (!loadWorkerIdFromFileWhenZkDown) { + throw e; + } + SnowflakeWorkerIdServiceImpl.log.error("can load worker id from zk , try to load worker id from file", e); + // 从本地文件中读取workerId，如果系统时针回调，可能会出现 + final Integer workerIdFromFile = loadWorkerIdFromFile(); + if (workerIdFromFile != null) { + workerId = workerIdFromFile; + return; + } + throw e; + } + } + + + private Integer getWorkerId(String localZKPath) { + return Integer.parseInt(localZKPath.split("-")[1]); + } + + /** + * @return true 检查通过 + */ + private void checkTimestamp(CuratorFramework client, String localZKPath) throws Exception { + final Endpoint endpoint = parseData(new String(client.getData().forPath(localZKPath))); + // 该节点的时间不能大于最后一次上报的时间 + if (endpoint.getTimestamp() > System.currentTimeMillis()) { + throw new SystemClockCallbackException("system clock callback"); + } + } + + /** + * 获取zk连接 + */ + public CuratorFramework getZKConnection() { + CuratorFramework framework = CuratorFrameworkFactory.builder() + .connectString(connectionString) + .retryPolicy(new RetryUntilElapsed((int) TimeUnit.SECONDS.toMillis(5), (int) TimeUnit.SECONDS.toMillis(1))) + .connectionTimeoutMs((int) TimeUnit.SECONDS.toMillis(10)) + .sessionTimeoutMs((int) TimeUnit.SECONDS.toMillis(6)) + .build(); + framework.start(); + return framework; + } + + /** + * 上传数据到zk,每5s上传一次 + */ + private void scheduledUploadTimeToZK(final CuratorFramework client, final String localZKPath) { + scheduledExecutorService.schedule(new Runnable() { + @Override + public void run() { + // 如果时针回调了就不同步 + if (System.currentTimeMillis() < lastUpdateAt) { + return; + } + try { + client.setData().forPath(localZKPath, buildData()); + lastUpdateAt = System.currentTimeMillis(); + log.debug("upload time to zk at" + lastUpdateAt); + } catch (Exception e) { + log.error("update init data error path is {} error is {}", localZKPath, e); + } + } + }, 5, TimeUnit.SECONDS); + } + + /** + * 将获取到的workerId持久化到文件 + * + * @return + */ + private Integer loadWorkerIdFromFile() { + try (InputStream resourceAsStream = this.getClass().getClassLoader().getResourceAsStream(DUMP_PATH)) { + Properties properties = new Properties(); + properties.load(resourceAsStream); + final String workerID = properties.getProperty("workerID"); + if (workerID != null) { + return Integer.parseInt(workerID); + } + return null; + } catch (IOException e) { + SnowflakeWorkerIdServiceImpl.log.error("load worker id from file error", e); + } + return null; + } + + private void updateWorkerId(int workerId) { + if (!loadWorkerIdFromFileWhenZkDown) { + return; + } + try { + String classpath = ResourceUtils.getURL("classpath:").getFile(); + File file = new File(classpath + "/" + DUMP_PATH); + if (!file.exists()) { + boolean mkdirs = file.getParentFile().mkdirs(); + if (!mkdirs) { + log.error("mkdir {} error", file.getParentFile().toString()); + return; + } + log.info("mkdir {}", file.toString()); + } + Files.write(file.toPath(), ("workerID=" + workerId).getBytes()); + } catch (FileNotFoundException e) { + log.error("", e); + } catch (IOException e) { + log.warn("write workerID to file {} error", DUMP_PATH, e); + } + } + + private String createPersistentSequentialNode(CuratorFramework client, String path, byte[] data) throws Exception { + return client.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.PERSISTENT_SEQUENTIAL) + .forPath(path + "-", data); + } + + private byte[] buildData() { + return (ip + "&" + port + "&" + System.currentTimeMillis()).getBytes(); + } + + private Endpoint parseData(String data) { + String[] split = data.split("&"); + return new Endpoint(split[0], split[1], Long.parseLong(split[2])); + } + + /** + * 上传到zk的数据 + */ + private static class Endpoint { + private String ip; + private String port; + private Long timestamp; + + public Endpoint(String ip, String port, Long timestamp) { + this.ip = ip; + this.port = port; + this.timestamp = timestamp; + } + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public Long getTimestamp() { + return timestamp; + } + + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } + } +} diff --git a/tinyid-server/src/main/resources/offline/application.properties b/tinyid-server/src/main/resources/offline/application.properties index 05bd436..6fb6d58 100755 --- a/tinyid-server/src/main/resources/offline/application.properties +++ b/tinyid-server/src/main/resources/offline/application.properties @@ -13,12 +13,15 @@ datasource.tinyid.primary.username=root datasource.tinyid.primary.password=123456 #datasource.tinyid.primary.testOnBorrow=false #datasource.tinyid.primary.maxActive=10 - #datasource.tinyid.secondary.driver-class-name=com.mysql.jdbc.Driver #datasource.tinyid.secondary.url=jdbc:mysql://localhost:3306/db2?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8 #datasource.tinyid.secondary.username=root #datasource.tinyid.secondary.password=123456 #datasource.tinyid.secondary.testOnBorrow=false #datasource.tinyid.secondary.maxActive=10 - - +# 是否启用zk +snowflake.enable=true +# zk连接信息 ip1:port1,ip2:port2 +snowflake.zk.connection-string:127.0.0.1:2181 +# 当zk不可访问时，从本地文件中读取之前备份的workerId +snowflake.zk.load-worker-id-from-file-when-zk-down=true \ No newline at end of file diff --git a/tinyid-server/src/main/resources/online/application.properties b/tinyid-server/src/main/resources/online/application.properties index 73ace54..e204939 100755 --- a/tinyid-server/src/main/resources/online/application.properties +++ b/tinyid-server/src/main/resources/online/application.properties @@ -11,8 +11,13 @@ datasource.tinyid.primary.driver-class-name=com.mysql.jdbc.Driver datasource.tinyid.primary.url=jdbc:mysql://localhost:3306/db1?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8 datasource.tinyid.primary.username=root datasource.tinyid.primary.password=123456 - #datasource.tinyid.secondary.driver-class-name=com.mysql.jdbc.Driver #datasource.tinyid.secondary.url=jdbc:mysql://localhost:3306/db2?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8 #datasource.tinyid.secondary.username=root #datasource.tinyid.secondary.password=123456 +# 是否启用zk +snowflake.enable=true +# zk连接信息 ip1:port1,ip2:port2 +snowflake.zk.connection-string:127.0.0.1:2181 +# 当zk不可访问时，从本地文件中读取之前备份的workerId +snowflake.zk.load-worker-id-from-file-when-zk-down=true \ No newline at end of file diff --git a/tinyid-server/src/test/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeWorkerIdServiceImplTest.java b/tinyid-server/src/test/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeWorkerIdServiceImplTest.java new file mode 100644 index 0000000..be11de9 --- /dev/null +++ b/tinyid-server/src/test/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeWorkerIdServiceImplTest.java @@ -0,0 +1,45 @@ +package com.xiaoju.uemc.tinyid.server.service.impl; + +import com.xiaoju.uemc.tinyid.server.service.SnowflakeWorkIdService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; + +/** + * @author zhangbingbing + * @date 2020/10/16 + */ +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) +public class SnowflakeWorkerIdServiceImplTest { + + @Autowired + private SnowflakeWorkIdService snowflakeWorkIdService; + + @Test + public void testWorkId() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(5); + final Set workIds = new HashSet<>(5); + for (int i = 0; i < 5; i++) { + new Thread(new Runnable() { + @Override + public void run() { + try { + workIds.add(snowflakeWorkIdService.workId()); + } finally { + latch.countDown(); + } + } + }).start(); + } + latch.await(); + Assert.assertEquals(1, workIds.size()); + } +} \ No newline at end of file From f8304269ee2520e1690fd4bca0a1ea35114f9d12 Mon Sep 17 00:00:00 2001 From: zhangbingbing Date: Fri, 16 Oct 2020 15:58:32 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=EF=BC=9A=E9=80=9A=E8=BF=87=E9=9B=AA=E8=8A=B1=E7=AE=97=E6=B3=95?= =?UTF-8?q?=E8=8E=B7=E5=8F=96id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/SnowflakeIdController.java | 67 +++++++++++++++++++ .../SnowflakeIdGeneratorFactoryServer.java | 31 +++++++++ .../server/service/SnowflakeIdService.java | 14 ++++ .../service/impl/SnowflakeIdServiceImpl.java | 31 +++++++++ 4 files changed, 143 insertions(+) create mode 100644 tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/controller/SnowflakeIdController.java create mode 100755 tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/factory/impl/SnowflakeIdGeneratorFactoryServer.java create mode 100644 tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/SnowflakeIdService.java create mode 100644 tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeIdServiceImpl.java diff --git a/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/controller/SnowflakeIdController.java b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/controller/SnowflakeIdController.java new file mode 100644 index 0000000..8d20444 --- /dev/null +++ b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/controller/SnowflakeIdController.java @@ -0,0 +1,67 @@ +package com.xiaoju.uemc.tinyid.server.controller; + +import com.xiaoju.uemc.tinyid.server.common.annotation.Module; +import com.xiaoju.uemc.tinyid.server.service.SnowflakeIdService; +import com.xiaoju.uemc.tinyid.server.service.TinyIdTokenService; +import com.xiaoju.uemc.tinyid.server.vo.ErrorCode; +import com.xiaoju.uemc.tinyid.server.vo.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** + * @author zhangbingbing + * @date 2020/10/16 + */ +@Module(value = "snowflake.enable") +@RestController +@RequestMapping("/id/snowflake") +public class SnowflakeIdController { + + private static final Logger logger = LoggerFactory.getLogger(SnowflakeIdController.class); + + @Autowired + private TinyIdTokenService tinyIdTokenService; + @Autowired + private SnowflakeIdService idService; + + @RequestMapping("/nextIds") + public Response> nextIds(String bizType, Integer batchSize, String token) { + Response> response = new Response<>(); + if (!tinyIdTokenService.canVisit(bizType, token)) { + response.setCode(ErrorCode.TOKEN_ERR.getCode()); + response.setMessage(ErrorCode.TOKEN_ERR.getMessage()); + return response; + } + try { + response.setData(idService.nextIdBatch(bizType, batchSize)); + } catch (Exception e) { + response.setCode(ErrorCode.SYS_ERR.getCode()); + response.setMessage(e.getMessage()); + logger.error("nextId error", e); + } + return response; + } + + @RequestMapping("/nextId") + public Response nextId(String bizType, String token) { + Response response = new Response<>(); + if (!tinyIdTokenService.canVisit(bizType, token)) { + response.setCode(ErrorCode.TOKEN_ERR.getCode()); + response.setMessage(ErrorCode.TOKEN_ERR.getMessage()); + return response; + } + try { + response.setData(idService.nextId(bizType)); + } catch (Exception e) { + response.setCode(ErrorCode.SYS_ERR.getCode()); + response.setMessage(e.getMessage()); + logger.error("nextId error", e); + } + return response; + } +} diff --git a/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/factory/impl/SnowflakeIdGeneratorFactoryServer.java b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/factory/impl/SnowflakeIdGeneratorFactoryServer.java new file mode 100755 index 0000000..3e8a1f2 --- /dev/null +++ b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/factory/impl/SnowflakeIdGeneratorFactoryServer.java @@ -0,0 +1,31 @@ +package com.xiaoju.uemc.tinyid.server.factory.impl; + +import com.xiaoju.uemc.tinyid.base.factory.AbstractIdGeneratorFactory; +import com.xiaoju.uemc.tinyid.base.generator.IdGenerator; +import com.xiaoju.uemc.tinyid.base.generator.impl.SnowflakeIdGenerator; +import com.xiaoju.uemc.tinyid.server.common.annotation.Module; +import com.xiaoju.uemc.tinyid.server.service.SnowflakeWorkIdService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * @author zhangbingbing + * @date 2020/10/16 + */ +@Component +@Module("snowflake.enable") +public class SnowflakeIdGeneratorFactoryServer extends AbstractIdGeneratorFactory { + + private static final Logger logger = LoggerFactory.getLogger(SnowflakeIdGeneratorFactoryServer.class); + + @Autowired + private SnowflakeWorkIdService workIdService; + + @Override + public IdGenerator createIdGenerator(String bizType) { + logger.info("SnowflakeIdGenerator :{}", bizType); + return new SnowflakeIdGenerator(workIdService.workId()); + } +} diff --git a/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/SnowflakeIdService.java b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/SnowflakeIdService.java new file mode 100644 index 0000000..7755c92 --- /dev/null +++ b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/SnowflakeIdService.java @@ -0,0 +1,14 @@ +package com.xiaoju.uemc.tinyid.server.service; + +import java.util.List; + +/** + * @author zhangbingbing + * @date 2020/10/16 + */ +public interface SnowflakeIdService { + + Long nextId(String businessType); + + List nextIdBatch(String businessType, int batchSize); +} diff --git a/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeIdServiceImpl.java b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeIdServiceImpl.java new file mode 100644 index 0000000..558486d --- /dev/null +++ b/tinyid-server/src/main/java/com/xiaoju/uemc/tinyid/server/service/impl/SnowflakeIdServiceImpl.java @@ -0,0 +1,31 @@ +package com.xiaoju.uemc.tinyid.server.service.impl; + +import com.xiaoju.uemc.tinyid.server.common.annotation.Module; +import com.xiaoju.uemc.tinyid.server.factory.impl.SnowflakeIdGeneratorFactoryServer; +import com.xiaoju.uemc.tinyid.server.service.SnowflakeIdService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @author zhangbingbing + * @date 2020/10/16 + */ +@Service +@Module(value = "snowflake.enable") +public class SnowflakeIdServiceImpl implements SnowflakeIdService { + + @Autowired + private SnowflakeIdGeneratorFactoryServer factoryServer; + + @Override + public Long nextId(String businessType) { + return factoryServer.getIdGenerator(businessType).nextId(); + } + + @Override + public List nextIdBatch(String businessType, int batchSize) { + return factoryServer.getIdGenerator(businessType).nextId(batchSize); + } +}