woggioni/jpacrepo
1
0
Fork 0
Code Actions Packages Releases Activity

fixed bug with computing MD5 hash and multithreading

Browse Source
This commit is contained in:
Walter Oggioni
2015-04-25 17:50:26 +02:00
parent db786b73c9
commit ee86dfe895
8 changed files with 172 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/main/java/org/jpacrepo/pacbase/Hasher.java
View File

@@ -1,9 +1,8 @@
package org.jpacrepo.pacbase;
import org.apache.commons.codec.binary.Hex;
import java.io.IOException;
import java.io.InputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -14,6 +13,8 @@ public class Hasher
{
static private final MessageDigest md5;
private MessageDigest md;
static
{
try
@@ -25,20 +26,44 @@ public class Hasher
}
}
public Hasher(String algorithm)
{
try
{
md = MessageDigest.getInstance(algorithm);
}
catch (NoSuchAlgorithmException e)
{
throw new RuntimeException(e);
}
}
public static String computeMD5(InputStream is) throws IOException
{
md5.reset();
byte[] buffer2 = new byte[1000000];
byte[] buffer = new byte[100000];
while (is.available() > 0)
{
int read = is.read(buffer2, 0, 1000000);
md5.update(buffer2,0,read);
int read = is.read(buffer, 0, 100000);
md5.update(buffer, 0, read);
}
is.close();
return bytesToHex(md5.digest());
}
final protected static char[] hexArray = "0123456789ABCDEF".toCharArray();
public String getHashString(InputStream is) throws IOException
{
byte[] buffer = new byte[100000];
while (is.available() > 0)
{
int read = is.read(buffer, 0, 100000);
md.update(buffer, 0, read);
}
is.close();
return bytesToHex(md.digest());
}
final private static char[] hexArray = "0123456789ABCDEF".toCharArray();
public static String bytesToHex(byte[] bytes)
{
@@ -51,4 +76,21 @@ public class Hasher
}
return new String(hexChars);
}
static MessageDigest getMd5()
{
try
{
return MessageDigest.getInstance("MD5");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
public static MD5InputStream createMD5InputStream(InputStream is)
{
return new MD5InputStream(is);
}
}

21
src/main/java/org/jpacrepo/pacbase/MD5InputStream.java Normal file
View File

@@ -0,0 +1,21 @@
package org.jpacrepo.pacbase;
import java.io.InputStream;
import java.security.DigestInputStream;
/**
* Created by walter on 25/04/15.
*/
public class MD5InputStream extends DigestInputStream
{
MD5InputStream(InputStream is)
{
super(is, Hasher.getMd5());
}
public String digest()
{
byte[] digest = getMessageDigest().digest();
return Hasher.bytesToHex(digest);
}
}

23
src/main/java/org/jpacrepo/pacbase/Parser.java
View File

@@ -17,12 +17,20 @@ import java.util.*;
public class Parser
{
public static PkgData parseFile(File file) throws Exception
private XZCompressorInputStream xzcis;
private TarArchiveInputStream tais;
private ArchiveEntry ae;
private Hasher hasher;
public Parser()
{
FileInputStream fis = new FileInputStream(file);
XZCompressorInputStream xzcis = new XZCompressorInputStream(fis);
TarArchiveInputStream tais = new TarArchiveInputStream(xzcis);
ArchiveEntry ae;
hasher = new Hasher("MD5");
}
public PkgData parseFile(File file) throws Exception
{
xzcis = new XZCompressorInputStream(new FileInputStream(file));
tais = new TarArchiveInputStream(xzcis);
while((ae = tais.getNextEntry()) != null)
{
if(ae.getName().equals(".PKGINFO"))
@@ -57,7 +65,6 @@ public class Parser
}
tais.close();
xzcis.close();
fis.close();
PkgData data = new PkgData();
for(String key : propMap.keySet())
@@ -118,17 +125,15 @@ public class Parser
case "pkgbase":
data.base = propMap.get(key).get(0);
break;
}
}
data.md5sum = Hasher.computeMD5(new FileInputStream(file));
data.md5sum = hasher.getHashString(new FileInputStream(file));
data.fileName = file.getName();
return data;
}
}
tais.close();
xzcis.close();
fis.close();
return null;
}

13
src/main/java/org/jpacrepo/service/PacmanServiceEJB.java
View File

@@ -108,12 +108,17 @@ public class PacmanServiceEJB implements PacmanService
}
}
private void parseFile(File file) throws Exception
private void parseFile2(Parser p, File file) throws Exception
{
}
private void parseFile(Parser p, File file) throws Exception
{
TypedQuery<PkgName> nquery = em.createQuery(nameQuery, PkgName.class);
TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
PkgData data = Parser.parseFile(file);
PkgData data = p.parseFile(file);
hquery.setParameter("md5sum", data.md5sum);
List<PkgData> savedFiles = hquery.getResultList();
if (savedFiles.size() > 0)
@@ -172,6 +177,8 @@ public class PacmanServiceEJB implements PacmanService
{
Deque<File> ls;
Parser parser = new Parser();
public SyncWorker(Deque<File> ls)
{
this.ls = ls;
@@ -193,7 +200,7 @@ public class PacmanServiceEJB implements PacmanService
if (p == null || file.lastModified() > p.updTimestamp.getTime())
{
ut.begin();
parseFile(file);
parseFile(parser, file);
ut.commit();
}
}

6
src/main/java/org/jpacrepo/service/PacmanWebService.java
View File

@@ -7,7 +7,6 @@ import org.jpacrepo.model.PkgData;
import org.jpacrepo.model.PkgList;
import org.jpacrepo.model.PkgName;
import org.apache.commons.io.IOUtils;
import org.jpacrepo.pacbase.Hasher;
import org.jpacrepo.pacbase.Parser;
import org.jpacrepo.persistence.QueryEngine;
@@ -20,7 +19,6 @@ import javax.ws.rs.core.Context;
import java.io.*;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -49,6 +47,8 @@ public class PacmanWebService
@DefaultConfiguration
private ApplicationContext ctx;
private Parser serviceParser = new Parser();
@GET
@Path("search")
public Response getPackage(@QueryParam("name") String name,
@@ -184,7 +184,7 @@ public class PacmanWebService
}
else
{
PkgData pkg = Parser.parseFile(file);
PkgData pkg = serviceParser.parseFile(file);
TypedQuery<PkgData> hquery = em.createQuery(hashQuery, PkgData.class);
hquery.setParameter("md5sum", pkg.md5sum);

4
src/main/resources/META-INF/persistence.xml
View File

@@ -7,8 +7,8 @@
<persistence-unit name="jpacrepo_pu" transaction-type="JTA">
<jta-data-source>java:/ejb/postgres/wildfly</jta-data-source>
<properties>
<property name="javax.persistence.schema-generation.database.action" value="drop-and-create"/>
<!--<property name="javax.persistence.schema-generation.database.action" value="none"/>-->
<!--<property name="javax.persistence.schema-generation.database.action" value="drop-and-create"/>-->
<property name="javax.persistence.schema-generation.database.action" value="none"/>
<!--<property name="javax.persistence.schema-generation.database.action" value="create"/>-->
<property name="eclipselink.logging.level" value="INFO"/>
<property name="hibernate.default_schema" value="jpacrepo"/>