opensource/timemachineplus
3
0
Fork 0
mirror of https://github.com/epcdiy/timemachineplus.git synced 2026-09-01 15:35:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

update to 1.0.1

Browse Source
This commit is contained in:
liuqiang1
2023-07-24 23:36:46 +08:00
parent 0b758d61b0
commit 0d72803a6f
20 changed files with 30 additions and 991 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
-2
View File
@@ -1,2 +0,0 @@
# timemachineplus
苹果timemachine复刻，超越，可支持本地磁盘数据和局域网拉取备份其他电脑，支持多备份硬盘分布式存储，java开发，全平台支持
pom.xml
+28 -43
View File
@@ -4,9 +4,11 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>epcdiy</groupId>
<groupId>org.epcdiy</groupId>
<artifactId>timemachineplus</artifactId>
<version>1.0-SNAPSHOT</version>
<version>1.0.1-SNAPSHOT</version>
<name>TimeMachine Plus</name>
<url>https://github.com/epcdiy/timemachineplus</url>
<dependencies>
<!-- https://mvnrepository.com/artifact/log4j/log4j -->
@@ -15,49 +17,13 @@
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>6.0.6</version>
</dependency>
<dependency>
<groupId>cn.com.infosec</groupId>
<artifactId>infosec</artifactId>
<version>1.0</version>
<scope>system</scope>
<systemPath>${basedir}/lib/crypto.jar</systemPath>
</dependency>
<dependency>
<groupId>cn.com.infosec</groupId>
<artifactId>infosec_isfj</artifactId>
<version>1.0</version>
<scope>system</scope>
<systemPath>${basedir}/lib/ISFJ_v2_0_139_7_BAISC_JDK14.jar</systemPath>
</dependency>
<dependency>
<groupId>cn.com.infosec.netsign</groupId>
<artifactId>netsign</artifactId>
<version>1.0</version>
<scope>system</scope>
<systemPath>${basedir}/lib/netsign.jar</systemPath>
</dependency>
<dependency>
<groupId>cn.com.infosec.netsign.agent</groupId>
<artifactId>agent</artifactId>
<version>1.0</version>
<scope>system</scope>
<systemPath>${basedir}/lib/netsignapi.jar</systemPath>
</dependency>
<dependency>
<groupId>cn.com.infosec.netsign</groupId>
<artifactId>infosec_fw</artifactId>
<version>1.0</version>
<scope>system</scope>
<systemPath>${basedir}/lib/nsframework.jar</systemPath>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
@@ -82,7 +48,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>RELEASE</version>
<version>1.18.14</version>
<scope>compile</scope>
</dependency>
@@ -90,17 +56,36 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.epcdiy.timemachineplus.ServiceRun</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
release/commons-codec-1.15.jar
BIN
View File
Binary file not shown.
release/commons-logging-1.1.3.jar
BIN
View File
Binary file not shown.
release/httpclient-4.5.5.jar
BIN
View File
Binary file not shown.
release/httpcore-4.4.9.jar
BIN
View File
Binary file not shown.
release/httpmime-4.5.5.jar
BIN
View File
Binary file not shown.
release/json-20180130.jar
BIN
View File
Binary file not shown.
release/log4j-1.2.17.jar
BIN
View File
Binary file not shown.
release/mysql-connector-java-6.0.6.jar
BIN
View File
Binary file not shown.
release/run.bat
+1 -1
View File
@@ -1 +1 @@
java -classpath "/程序所在路径/*" serviceRun
java -jar timemachineplus-1.0.1-SNAPSHOT.jar
release/run.sh
+1 -1
View File
@@ -1 +1 @@
java -classpath "/程序所在路径/*" serviceRun
java -jar timemachineplus-1.0.1-SNAPSHOT.jar
release/timemachineplus-1.0-SNAPSHOT.jar
BIN
View File
Binary file not shown.
src/main/java/BackupHistory.java
-14
View File
@@ -1,14 +0,0 @@
import lombok.Data;
@Data
public class BackupHistory
{
int id;
long backupfileid;
long filesize;
long motifytime;
String backuptargetpath;
String backuptargetfullpath;
int backuptargetrootid;
String md5;
}
src/main/java/Backuproot.java
-8
View File
@@ -1,8 +0,0 @@
import lombok.Data;
@Data
public class Backuproot
{
int id;
String rootpath;
}
src/main/java/Backuptargetroot.java
-8
View File
@@ -1,8 +0,0 @@
import lombok.Data;
@Data
public class Backuptargetroot {
int id;
String tagetrootpath;
String targetrootdir;
long spaceRemain;
}
src/main/java/HttpClintTool.java
-149
View File
@@ -1,149 +0,0 @@
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.FileEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.FileBody;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
import org.apache.http.message.BasicHeader;
import org.apache.http.protocol.HTTP;
import org.apache.http.ssl.SSLContexts;
import org.apache.http.util.CharsetUtils;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import javax.net.ssl.*;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.KeyStore;
import java.security.SecureRandom;
import java.security.cert.Certificate;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.util.Enumeration;
import java.util.Map;
import java.util.ResourceBundle;
import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
* Created by chenjunlai on 2018/5/10
*/
public class HttpClintTool {
static Logger log = Logger.getLogger(HttpClintTool.class);
/**
* 证书密码
*/
private static String clientCertPassword = "123456";
private HttpClient httpClient;
public HttpClintTool() throws Exception{
// HttpHost proxy = new HttpHost("192.168.13.19", 7777);
//DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
httpClient = HttpClients.custom()
// .setRoutePlanner(routePlanner)
.setConnectionTimeToLive(15, TimeUnit.SECONDS)
.build();
/*
StringEntity se = new StringEntity("1234");
se.setContentType("text/json");
se.setContentEncoding(new BasicHeader(HTTP.CONTENT_TYPE, "application/json"));
URI uri =new URI("https://222.76.227.136:18080/qrcode_CAVerify/");
log.info("post地址:"+uri);
HttpPost httpPost = new HttpPost(uri);
// httpPost.setEntity(reqEntity);
//httpPost.addHeader("Cookie", cookie);
httpPost.setEntity(se);
HttpResponse response = httpClient.execute(httpPost);
String resultString = null;
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
resultString = EntityUtils.toString(response.getEntity());
}
log.info("POST返回值 ： "+response.getStatusLine().getStatusCode()+", 返回内容："+resultString+",response: "+response );
*/
}
/**
*
* @param remotePath 远程地址
* @return
* @throws URISyntaxException
* @throws ClientProtocolException
* @throws IOException
*/
public String post(String remotePath, String jsonstr)
throws URISyntaxException, ClientProtocolException, IOException {
/* Set<String> keys = args.keySet();
URIBuilder https = new URIBuilder().setScheme("https").setHost(host).setPort(port).setPath(remotePath);
for (String key : keys) {
https.setParameter(key,args.get(key));
}
URI uri = https.build();
log.info("post地址:{}",uri);
HttpPost httpPost = new HttpPost(uri);
FileEntity fileEntity = new FileEntity(new File(filename));
fileEntity.setChunked(true);
httpPost.setEntity(fileEntity);
httpPost.addHeader("Cookie", cookie);
*/
/*
// 把文件转换成流对象FileBody
File localFile = new File(localPath);
FileBody fileBody = new FileBody(localFile);
// 以浏览器兼容模式运行，防止文件名乱码。
HttpEntity reqEntity = MultipartEntityBuilder.create()
.setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
.addPart(filename, fileBody)
.setCharset(CharsetUtils.get("UTF-8")).build();
// uploadFile对应服务端类的同名属性<File类型>
// .addPart("uploadFileName", uploadFileName)
// uploadFileName对应服务端类的同名属性<String类型>
*/
StringEntity se = new StringEntity(jsonstr);
se.setContentType("application/json;charset=UTF-8");
se.setContentEncoding(new BasicHeader(HTTP.CONTENT_TYPE, "application/json"));
URI uri =new URI(remotePath);
log.info("post地址:"+uri);
HttpPost httpPost = new HttpPost(uri);
// httpPost.setEntity(reqEntity);
//httpPost.addHeader("Cookie", cookie);
httpPost.setEntity(se);
HttpResponse response = httpClient.execute(httpPost);
String resultString = null;
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
resultString = EntityUtils.toString(response.getEntity());
}
log.info("POST返回值 ： "+response.getStatusLine().getStatusCode()+", 返回内容："+resultString+",response: "+response );
return resultString;
}
}
src/main/java/MySQLHelper.java
-254
View File
@@ -1,254 +0,0 @@
import org.apache.log4j.Logger;
import java.io.*;
import java.sql.*;
import java.util.ResourceBundle;
/**
*
* 项目名称：JavaSQL1
* 类名称：DBHelper
* 类描述：MySQL数据库操作类
* 创建人：Administrator
* 创建时间：2014-11-25 下午5:11:11
* 修改备注：
* @version
*/
public class MySQLHelper
{
public String url = null;// "jdbc:mysql://127.0.0.1/vge_whu"; //数据库连接
public String name = "com.mysql.jdbc.Driver"; //程序驱动
public String user = null;//"root"; //用户名
public String password = null;//"abc@123"; //密码
private String configFilename;
public int maxpackage=1024;
public Connection conn = null;
private Statement statement=null;
static Logger logger = Logger.getLogger(MySQLHelper.class);
private void initCfg() throws Exception {
ResourceBundle bundle=null;
try {
bundle = ResourceBundle.getBundle(configFilename);
url=bundle.getString("mysqlurl");
user=bundle.getString("mysqluser");
password=bundle.getString("mysqlpassword");
maxpackage=Integer.parseInt(bundle.getString("max_allowed_packet"));
} catch (Exception var6) {
var6.printStackTrace(System.out);
logger.error("Can not Found Config File dbconfig.properties");
}
}
public boolean init()
{
if(url==null)
{
try {
initCfg();
} catch (Exception e) {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
logger.error(e.toString());
return false;
}
}
try
{
Class.forName(name);// 指定连接类型
conn = DriverManager.getConnection(url, user, password);// 获取连接
logger.info("mysql connected!");
return true;
} catch (Exception e)
{
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
logger.error(e.toString());
}
return false;
}
/**
*
* 创建一个新的实例 DBHelper.
*/
public MySQLHelper(String dbconfig)
{
configFilename=dbconfig;
init();
// importSql("D:\\project\\03_fj_tobacco_qr_code\\05_RD_TS_PI_VER_VAL\\02_概要设计\\02-接口设计\\数据上传和云端接收接口\\测试用文件\\db_180316141201\\tb_cigar.sql");
}
public boolean exeSql(String sql)
{
try {
if(!conn.isValid(1000))
{
if(!init())
{
return false;
}
}
conn.setAutoCommit(true);
PreparedStatement pst = conn.prepareStatement(sql);// 准备执行语句
pst.execute();
pst.close();
return true;
} catch (SQLException e)
{
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
logger.error(e.toString());
if(sql.length()>256)
{
logger.error(sql.substring(0, 256)+"...");
}
else
{
logger.error(sql);
}
}
return false;
}
public boolean importSql(String filePath)
{
logger.info("sql importing "+filePath);
StringBuilder sqlline=new StringBuilder("");
File file = new File(filePath);
if(file.exists()&&file.isFile())
{
int curpackage=0;
int cnt=0;
try
{
if(!conn.isValid(1000))
{
if(!init())
{
return false;
}
}
conn.setAutoCommit(false);
Statement stmt = conn.createStatement();
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), "UTF-8");// 考虑到编码格式
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = "";
sqlline=new StringBuilder("");
while ((lineTxt = bufferedReader.readLine()) != null)
{
sqlline.append(lineTxt);
if(sqlline.indexOf(";")!=-1)
{
stmt.addBatch(sqlline.toString());
curpackage+=sqlline.length();
if(curpackage>maxpackage)
{
curpackage=0;
logger.debug("executeBatch:"+stmt.executeBatch().length);
conn.commit();
}
sqlline=new StringBuilder("");
cnt++;
}
}
bufferedReader.close();
read.close();
if(curpackage>0)
{
curpackage=0;
logger.debug("final executeBatch:"+stmt.executeBatch().length);
conn.commit();
}
logger.info("imported "+cnt+" lines");
conn.setAutoCommit(true);
stmt.close();
return true;
}
catch (Exception e)
{
logger.error("file "+filePath+" read error!");
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
logger.error(e.toString());
if(sqlline.length()>256)
{
logger.error(sqlline.substring(0, 256)+"...");
}
else
{
logger.error(sqlline);
}
}
}
else
{
logger.error("file "+filePath+" unaccessable!");
}
return false;
}
public ResultSet querySql(String sql)
{
try {
if(!conn.isValid(1000))
{
if(!init())
{
return null;
}
}
conn.setAutoCommit(true);
if(statement!=null)
{
statement.close();
}
statement=conn.createStatement();
ResultSet rs = statement.executeQuery(sql);
return rs;
} catch (SQLException e) {
logger.error("querySql error!sql="+sql);
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
logger.error(e.toString());
logger.error(sql);
}
return null;
}
/**
*
* 方法名称: close ；
* 方法描述: 关闭数据库连接 ；
* 参数 ：
* 返回类型: void ；
* 创建人：James；
* 创建时间：2014-11-25 下午7:00:12；
* @throws
*/
public void close()
{
try
{
this.conn.close();
} catch (SQLException e)
{
e.printStackTrace();
}
}
}
src/main/java/serviceRun.java
-487
View File
@@ -1,487 +0,0 @@
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.apache.log4j.Level;
import java.io.*;
import java.nio.channels.FileChannel;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
/**
* Created by liuq2 on 2018/3/19.
*/
public class serviceRun {
static Logger logger = Logger.getLogger(serviceRun.class);
private MySQLHelper mysqlHelper;
private HttpClintTool myclient;
private List<Backuproot> BackupRootList=new ArrayList<>();
private List<Backuptargetroot> BackupTargetRootList=new ArrayList<>();
private long filecopycount=0;
private long datacopycount=0;
private int backupid=0;
final static String targetBkDirName="BACKUPDATABASE";
private Random random;
public void init()
{
logger.info("init db...");
PropertyConfigurator.configure ( this.getClass().getClassLoader().getResourceAsStream("log4j.properties"));
// Set the logger level to Level.INFO
logger.setLevel(Level.INFO);
mysqlHelper=new MySQLHelper("dbconfig");
try {
myclient=new HttpClintTool();
} catch (Exception e) {
e.printStackTrace();
}
long seed=System.nanoTime();
random=new Random(seed);
logger.info("using rand seed :"+seed);
}
public void loadBackupRoot() throws SQLException
{
logger.info("loadBackupRoot");
ResultSet ret=mysqlHelper.querySql("select * from tb_backuproot");
while(ret!=null&&ret.next())
{
Backuproot backuproot=new Backuproot();
backuproot.setId(ret.getInt("id"));
backuproot.setRootpath(ret.getString("rootpath"));
File checkfile=new File(backuproot.getRootpath());
if(!checkfile.exists())
{
logger.error("No found backup source :"+backuproot.getRootpath());
}
else {
BackupRootList.add(backuproot);
}
}
ret=mysqlHelper.querySql("select * from tb_backuptargetroot");
while(ret!=null&&ret.next())
{
Backuptargetroot backuptargetroot=new Backuptargetroot();
backuptargetroot.setId(ret.getInt("id"));
backuptargetroot.setTagetrootpath(ret.getString("tagetrootpath"));
backuptargetroot.setTargetrootdir(targetBkDirName);
File file=new File(backuptargetroot.getTagetrootpath());
if(!file.exists()) {
file.mkdir();
}
File checkfile=new File(backuptargetroot.getTagetrootpath());
//if(false&&!checkfile.exists())//removed after debug
if(!checkfile.exists())
{
logger.error("No found backup target :"+backuptargetroot.getTagetrootpath());
}
else {
backuptargetroot.setSpaceRemain(file.getFreeSpace());
BackupTargetRootList.add(backuptargetroot);
}
}
}
private static void loadAllFiles(File file,List<String> fileList)
{
File[] fs = file.listFiles();
for(File f:fs)
{
if(f.toString().indexOf("/.")!=-1||f.toString().indexOf("\\.")!=-1)
{
continue;
}
if(f.isDirectory()) //若是目录，则递归打印该目录下的文件
loadAllFiles(f,fileList);
if(f.isFile()) //若是文件
{
fileList.add(f.toString());
}
}
}
private Backuptargetroot getAvalidTarget(long needspace)
{
List<Backuptargetroot> avaliedTarget=new ArrayList<>();
for (Backuptargetroot backupTargetRoot : BackupTargetRootList)
{
File file=new File(backupTargetRoot.getTagetrootpath());
backupTargetRoot.setSpaceRemain(file.getFreeSpace());
if(backupTargetRoot.getSpaceRemain()>needspace)
{
avaliedTarget.add(backupTargetRoot);
//return backupTargetRoot;
}
}
if(avaliedTarget.size()>0)
{
Backuptargetroot backuptargetroot=avaliedTarget.get(0);
for(Backuptargetroot backuptargetroot1:avaliedTarget)
{
if(backuptargetroot1.getSpaceRemain()>backuptargetroot.getSpaceRemain())
{
backuptargetroot=backuptargetroot1;
}
}
return backuptargetroot;
}
return null;
}
private static void copyFileUsingFileChannels(File source, File dest) throws IOException {
FileChannel inputChannel = null;
FileChannel outputChannel = null;
try {
inputChannel = new FileInputStream(source).getChannel();
outputChannel = new FileOutputStream(dest).getChannel();
outputChannel.transferFrom(inputChannel, 0, inputChannel.size());
} finally {
inputChannel.close();
outputChannel.close();
}
}
boolean exeCopy(File fileHandle,String file,long backupfileid) throws IOException {
Backuptargetroot backuptargetroot=getAvalidTarget(fileHandle.length());
if(backuptargetroot==null)
{
logger.error("no space in all targetbackups! need:"+fileHandle.length());
return false;
}
String md5str=DigestUtils.md5Hex(new FileInputStream(file));
String targetName=backuptargetroot.getTagetrootpath()+File.separator+backuptargetroot.getTargetrootdir()+File.separator+md5str+"_"+System.currentTimeMillis();
String targetNameSave=File.separator+backuptargetroot.getTargetrootdir()+File.separator+md5str+"_"+System.currentTimeMillis();
LocalDateTime begincopysingle=LocalDateTime.now();
try {
copyFileUsingFileChannels(fileHandle, new File(targetName));
}
catch (Exception e)
{
logger.error("failed to copy file from "+file+" to "+targetName);
return false;
}
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
mysqlHelper.exeSql("insert into tb_backfilehistory (backupfileid,backupid,motifytime,filesize,copystarttime,copyendtime,backuptargetpath,backuptargetrootid,md5) values (" +
backupfileid+","+backupid+","+fileHandle.lastModified()+","+fileHandle.length()+",'"+dateTimeFormatter.format(begincopysingle)+"','"+dateTimeFormatter.format(LocalDateTime.now())+"','"+targetNameSave+"',"+backuptargetroot.getId()+",'"+md5str+"')");
logger.info("copy file from "+file+" to "+targetName);
return true;
}
private void XCopy(Backuproot backuproot) throws SQLException, IOException {
logger.info("Loading File list:"+backuproot.getRootpath());
List<String> fileList=new ArrayList<>();
Map<String,Long> mapFile=new HashMap<>();
loadAllFiles(new File(backuproot.getRootpath()),fileList);
logger.info("Loading CurrentFile In Db:"+ backuproot.getId());
{
String sqlquery = "select id,filepath from tb_backfiles where backuprootid=" + backuproot.getId();
ResultSet ret = mysqlHelper.querySql(sqlquery);
while(ret!=null&&ret.next())
{
mapFile.put(ret.getString("filepath"),ret.getLong("id"));
}
}
logger.info("begin xcopy! total:"+ fileList.size());
long counter=0;
long timestamp=new Date().getTime()/1000;
for(String file:fileList)
{
if(file.indexOf("/.")!=-1||file.indexOf("\\.")!=-1)
{
//不拷贝.开头的文件夹和文件
continue;
}
counter++;
if(new Date().getTime()/1000!=timestamp)
{
timestamp=new Date().getTime()/1000;
System.out.println("copy progress:"+counter*100/fileList.size()+"% "+counter+"/"+fileList.size());
}
File fileHandle=new File(file);
long id=0;
if(!mapFile.containsKey(file))
{
String sqlquery="select * from tb_backfiles where filepath='"+file.replace("\\","\\\\").replace("'","\\'")+"' and backuprootid="+backuproot.getId();
//ResultSet ret= mysqlHelper.querySql(sqlquery);
//if(ret==null||!ret.next()) {
//第一次拷贝，创建档案
mysqlHelper.exeSql("insert into tb_backfiles (backuprootid,filepath,versionhistorycnt,lastbackuptime) values (" +
backuproot.getId() + ",'" + file.replace("\\","\\\\").replace("'","\\'") + "',0,now())");
ResultSet ret= mysqlHelper.querySql(sqlquery);
if(ret==null||!ret.next())
{
logger.error("内部错误！数据库异常，退出...");
break;
}
id=ret.getLong("id");
}
else
{
id=mapFile.get(file);
}
//判断文件是否有修改
ResultSet ret=mysqlHelper.querySql("select * from tb_backfilehistory where backupfileid="+id+" order by id desc limit 1");
if(ret!=null&&ret.next())
{
long lastmotify=ret.getLong("motifytime");
long filesize=ret.getLong("filesize");
String hash=ret.getString("md5");
long fidid=ret.getLong("id");
if(lastmotify==fileHandle.lastModified()&&filesize==fileHandle.length())
{
//mysqlHelper.exeSql("insert into tb_backfilehistory (backupfileid,backupid,motifytime,filesize,backuptargetpath,backuptargetrootid,md5) values (" +
// id+","+backupid+","+fileHandle.lastModified()+","+fileHandle.length()+",'"+ret.getString("backuptargetpath")+"',"+ret.getInt("backuptargetrootid")+",'"+ret.getString("md5")+"')");
continue;
}
System.out.println("motify time indb:"+lastmotify+" real:"+fileHandle.lastModified()+" filesize indb:"+filesize+" real:"+fileHandle.length());
//如果修改时间不一样，文件大小一样，追加校验一次hash，如果hash一样，则更新修改时间，不执行备份
if(lastmotify!=fileHandle.lastModified()&&filesize==fileHandle.length())
{
String md5str=DigestUtils.md5Hex(new FileInputStream(fileHandle));
if(md5str.equals(hash))
{
logger.error("historyfile id=【"+fidid+"】 backupid:【"+id+"】 not changed but motifytime diff db：【"+lastmotify+"】 file system:【"+fileHandle.lastModified()+"】, correcting...");
fileHandle.setLastModified(lastmotify);
//mysqlHelper.exeSql("update tb_backfilehistory set motifytime="+lastmotify+" where backupfileid="+id+" and id="+fidid);
continue;
}
System.out.println("hash indb:"+hash+" real:"+md5str);
}
}
if(!exeCopy(fileHandle,file, id))
{
logger.error("拷贝错误！退出...");
break;
}
filecopycount++;
datacopycount+=fileHandle.length();
}
}
private int beginbackup() throws SQLException {
mysqlHelper.exeSql("insert into tb_backup (begintime) values(now())");
ResultSet ret=mysqlHelper.querySql("select LAST_INSERT_ID() from tb_backup");
if(ret!=null&&ret.next())
{
return ret.getInt("LAST_INSERT_ID()");
}
return -1;
}
private void finishbackup()
{
mysqlHelper.exeSql("update tb_backup set endtime=now(),filecopycount="+filecopycount+",datacopycount="+datacopycount+" where id="+backupid);
}
public void deleteByBackuprootid(long rootid) throws SQLException {
long counter = 0;
long timestamp = new Date().getTime() / 1000;
System.out.println("loading files backuprootid=" + rootid);
ResultSet ret = mysqlHelper.querySql("select id from tb_backfiles where backuprootid=" + rootid);
List<Long> tbbackfilesList = new ArrayList<>();
while (ret != null && ret.next())
{
tbbackfilesList.add(ret.getLong("id"));
}
for(long id:tbbackfilesList)
{
ResultSet subret= mysqlHelper.querySql("select id,backuptargetpath from tb_backfilehistory where backupfileid="+id);
if(subret!=null&&subret.next())
{
File file=new File(subret.getString("backuptargetpath"));
if(file.exists())
{
file.delete();
}
else
{
logger.error("not found target path:"+subret.getString("backuptargetpath"));
}
mysqlHelper.exeSql("delete from tb_backfilehistory where id="+subret.getInt("id"));
}
counter++;
if(new Date().getTime()/1000!=timestamp)
{
timestamp=new Date().getTime()/1000;
System.out.println("delete progress:"+counter*100/tbbackfilesList.size()+"% "+counter+"/"+tbbackfilesList.size());
}
}
mysqlHelper.exeSql("delete from tb_backfiles where backuprootid="+rootid);
}
public void XCopy()
{
try {
backupid=beginbackup();
if ( backupid== -1) {
return;
}
}
catch (Exception e)
{
logger.error(e.getMessage());
return;
}
for(Backuproot backuproot:BackupRootList)
{
try {
XCopy(backuproot);
}
catch (Exception e)
{
logger.error(e.getMessage());
}
}
finishbackup();
}
private String getTargetrootPath(int targetbkid)
{
for(Backuptargetroot backuptargetroot:BackupTargetRootList)
{
if(backuptargetroot.getId()==targetbkid)
{
return backuptargetroot.getTagetrootpath();
}
}
return null;
}
private void removeWastedData(long backupfilehistoryid,String backupfilefullpath) {
//get fileid
try {
ResultSet ret=mysqlHelper.querySql("select backupfileid from tb_backfilehistory where id="+backupfilehistoryid);
if(ret!=null&&ret.next())
{
long backupfileid=ret.getLong("backupfileid");
mysqlHelper.exeSql("delete from tb_backfilehistory where id="+backupfilehistoryid);
File file=new File(backupfilefullpath);
if(file.exists()) {
logger.info("delete broken file:"+backupfilefullpath);
file.delete();
}
ret=mysqlHelper.querySql("select count(*) from tb_backfilehistory where backupfileid="+backupfileid);
if(ret!=null&&ret.next())
{
int historycnt=ret.getInt("count(*)");
if(historycnt==0)
{
//delete parent
mysqlHelper.exeSql("delete from tb_backfiles where id="+backupfileid);
}
}
}
} catch (Exception e)
{
logger.error(e.getMessage());
}
}
private void checkdata(boolean withhash)
{
try {
logger.info("loading all file and check");
int innercounter=0;
List<BackupHistory> historyList=new ArrayList<>();
long timestamp=new Date().getTime()/1000;
while(true)
{
int counter=0;
ResultSet ret=mysqlHelper.querySql("select * from tb_backfilehistory limit "+innercounter+",1000");
while(ret!=null&&ret.next()) {
BackupHistory backupHistory = new BackupHistory();
backupHistory.setId(ret.getInt("id"));
backupHistory.setMd5(ret.getString("md5"));
backupHistory.setFilesize(ret.getLong("filesize"));
backupHistory.setBackupfileid(ret.getLong("backupfileid"));
backupHistory.setBackuptargetrootid(ret.getInt("backuptargetrootid"));
backupHistory.setBackuptargetpath(ret.getString("backuptargetpath"));
//check file
String backuprootpath = getTargetrootPath(backupHistory.getBackuptargetrootid()) + backupHistory.getBackuptargetpath();
backupHistory.backuptargetfullpath=backuprootpath;
File fp = new File(backuprootpath);
if (!fp.exists() || fp.length() != backupHistory.getFilesize()) {
//kill
//logger.info("file not found or size mismatch:"+backuprootpath);
historyList.add(backupHistory);
}
else if (withhash)
{
String md5str=DigestUtils.md5Hex(new FileInputStream(backuprootpath));
if(!backupHistory.getMd5().equals(md5str))
{
//kill
logger.info("file hash not mismatch:"+backuprootpath);
historyList.add(backupHistory);
}
}
counter++;
if(new Date().getTime()/1000!=timestamp)
{
timestamp=new Date().getTime()/1000;
System.out.println("check num:"+(innercounter+counter)+" found:"+historyList.size());
}
}
if(counter==0)
{
break;
}
innercounter+=counter;
}
innercounter=0;
logger.info("begin removing wasted backup file recoreds");
for(BackupHistory backupHistory:historyList)
{
innercounter++;
removeWastedData(backupHistory.getId(),backupHistory.getBackuptargetfullpath());
if(new Date().getTime()/1000!=timestamp)
{
timestamp=new Date().getTime()/1000;
System.out.println("rm db count:"+innercounter+" / "+innercounter*100/historyList.size()+"%");
}
}
}
catch (Exception e)
{
logger.error(e.getMessage());
}
}
public static void main(String[] args)
{
serviceRun serviceRun=new serviceRun();
try {
serviceRun.init();
serviceRun.loadBackupRoot();
if(args.length==1)
{
if(args[0].equals("checkdata"))
{
logger.info("begin to checkdata");
serviceRun.checkdata(false);
}
else if(args[0].equals("checkdatawithhash"))
{
logger.info("begin to checkdata with hash");
serviceRun.checkdata(true);
}
else {
logger.info("cleaning data from backuprootid：" + args[0]);
serviceRun.deleteByBackuprootid(Long.parseLong(args[0]));
return;
}
}
serviceRun.XCopy();
}
catch (Exception e)
{
e.printStackTrace();
}
}
}
timemachineplus.iml
-24
View File
@@ -1,24 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
<orderEntry type="library" name="Maven: mysql:mysql-connector-java:6.0.6" level="project" />
<orderEntry type="library" name="Maven: org.json:json:20180130" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.9" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.10" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpmime:4.5.5" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />
<orderEntry type="library" name="Maven: org.projectlombok:lombok:1.18.28" level="project" />
</component>
</module>