From 38d3022be0468e27a248c4b1b9025de772785a03 Mon Sep 17 00:00:00 2001 From: Walter Oggioni Date: Sat, 30 Sep 2017 00:49:35 +0200 Subject: [PATCH] added HTTP cache and package size --- nim/htmlutils | 2 +- nim/jpacrepo.nim | 95 ++++++++++---- .../jpacrepo/service/PacmanWebService.java | 118 +++++++++++++----- 3 files changed, 159 insertions(+), 56 deletions(-) diff --git a/nim/htmlutils b/nim/htmlutils index 366e6f0..12868f4 160000 --- a/nim/htmlutils +++ b/nim/htmlutils @@ -1 +1 @@ -Subproject commit 366e6f0be1c4b448345b32a37fad344b756b4624 +Subproject commit 12868f49c8a8755d9f82829a4cb1df9efd6fcc38 diff --git a/nim/jpacrepo.nim b/nim/jpacrepo.nim index a2589b7..10f75e0 100644 --- a/nim/jpacrepo.nim +++ b/nim/jpacrepo.nim @@ -11,11 +11,16 @@ var pkgMap : JsonNode const serverURL {.strdefine.}: string = "http://oggio88.soon.it/jpacrepo/" +proc last[T](s : seq[T]) : T = s[s.len - 1] +proc formatByteSize(size : BiggestInt) : string = size.float64.formatEng(precision=1, siPrefix=true, unit = "B") + type DownloadPanel = ref object badge : Node listgroup : Node pkgs : HashSet[string] footer : Node + sizeLabel : Node + size : BiggestInt proc newDownloadPanel(parent : Node) : DownloadPanel = let dp = DownloadPanel() @@ -45,6 +50,13 @@ proc newDownloadPanel(parent : Node) : DownloadPanel = "text-align" : "right", "display" :"none" } + "span": + style { + "font-weight" : "bold", + "padding-right" : "10px" + } + cb: + dp.sizeLabel = node "button": class ["btn", "btn-primary"] attrs {"type" : "button"} @@ -77,29 +89,49 @@ proc updateBadge(dp : DownloadPanel) = dp.footer.style.display = st dp.badge.style.display = st dp.badge.textContent = $dp.pkgs.len() - + +proc updateSize(dp : DownloadPanel) = + dp.sizeLabel.textContent = "Total: " & dp.size.formatByteSize + proc addPkg(dp : DownloadPanel, pkgfile : string) = if not dp.pkgs.contains(pkgfile): dp.pkgs.incl(pkgfile) - htmlDocument(document, dp.listgroup): - "li": - var listElement : Node - class ["list-group-item"] - text pkgfile - "span": - class ["glyphicon", "glyphicon-remove", "pull-right"] + let req = newXMLHTTPRequest() + let load_cb = proc(e : Event) = + let sz = parseInt(req.responseText) + dp.size += sz + dp.updateSize + htmlDocument(document, dp.listgroup): + "li": + var listElement : Node + class ["list-group-item"] + text pkgfile + "span": + class ["glyphicon", "glyphicon-remove", "pull-right"] + cb: + let fn = proc(e : Event) = + dp.pkgs.excl(pkgfile) + listElement.remove() + dp.updateBadge + dp.size -= sz + dp.updateSize + node.addEventListener("click", fn) cb: - let fn = proc(e : Event) = - dp.pkgs.excl(pkgfile) - listElement.remove() - dp.updateBadge - node.addEventListener("click", fn) - cb: - listElement = node - + listElement = node + req.addEventListener("load", load_cb) + req.open("get", serverURL & "rest/pkg/filesize/" & pkgfile) + req.setRequestHeader("Accept", "application/json") + req.send() dp.updateBadge +proc readTableRow(row : Node) : JsonNode = + let pkgname = $row.querySelector("td:nth-child(2)").textContent + let version = $row.querySelector("td:nth-child(3) button").textContent + let arch = $row.querySelector("td:nth-child(4) button").textContent + pkgMap[pkgname][version][arch] + proc createDropdown(parent : Node, data :seq[string], onchange : proc(value : string)) = + htmlDocument(document, parent): "div": var button : Node @@ -108,9 +140,7 @@ proc createDropdown(parent : Node, data :seq[string], onchange : proc(value : st class ["btn", "btn-default", "dropdown-toggle"] attrs {"data-toggle": "dropdown", "type": "button"} cb: - for line in data: - node.textContent = line - break + node.textContent = data.last button = node "ul": class ["dropdown-menu"] @@ -162,6 +192,8 @@ proc newPkgTable(parent: Node, searchString : string) : PkgTable = text "Version" "th": text "Arch" + "th": + text "Installed size" "tbody": cb: var i = 0 @@ -169,7 +201,12 @@ proc newPkgTable(parent: Node, searchString : string) : PkgTable = closureScope: htmlDocument(document, node): "tr": + var row : Node var archCell : Node + var sizeCell : Node + let size_change_callback = proc(newValue : string) = + sizeCell.textContent = readTableRow(row)["size"].getNum.formatByteSize + "td": "div": class ["checkbox"] @@ -189,7 +226,8 @@ proc newPkgTable(parent: Node, searchString : string) : PkgTable = var newdata = newSeq[string]() for arch, pkgname in vs[newValue]: newdata.add(arch) - createDropdown(archCell, newdata, proc(s : string) = discard) + createDropdown(archCell, newdata, size_change_callback) + size_change_callback(newValue) createDropdown(node, data, change_callback) "td": cb: @@ -200,7 +238,17 @@ proc newPkgTable(parent: Node, searchString : string) : PkgTable = arches = a for arch, pkgname in arches: data.add(arch) - createDropdown(node, data, proc(s : string) = discard) + createDropdown(node, data, size_change_callback) + "td": + cb: + sizeCell = node + for v, arches in versions: + for key, value in arches: + node.textContent = value["size"].getNum.formatByteSize + return + cb: + row = node + pkgtable var dp : DownloadPanel @@ -248,10 +296,7 @@ htmlDocument document, document.body: for row in rows: let cbox = row.querySelector("td:first-child input") if cbox.checked: - let pkgname = $row.querySelector("td:nth-child(2)").textContent - let version = $row.querySelector("td:nth-child(3) button").textContent - let arch = $row.querySelector("td:nth-child(4) button").textContent - dp.addPkg(pkgMap[pkgname][version][arch].getStr) + dp.addPkg(readTableRow(row)["filename"].getStr) proc oninput(e : Event) = if pkgMap.len == 0 or node.value.len < 2: return let pkgtable = newPkgTable(table, $node.value) diff --git a/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java b/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java index eb84652..afce945 100644 --- a/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java +++ b/src/main/java/com/oggio88/jpacrepo/service/PacmanWebService.java @@ -8,12 +8,13 @@ import com.oggio88.jpacrepo.model.PkgList; import com.oggio88.jpacrepo.model.PkgName; import com.oggio88.jpacrepo.model.StringList; import com.oggio88.jpacrepo.pacbase.Parser; -import com.oggio88.jpacrepo.persistence.QueryEngine; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import javax.annotation.Resource; -import javax.ejb.*; +import javax.ejb.Singleton; +import javax.ejb.TransactionManagement; +import javax.ejb.TransactionManagementType; import javax.inject.Inject; import javax.persistence.*; import javax.persistence.criteria.CriteriaBuilder; @@ -24,6 +25,7 @@ import javax.transaction.Status; import javax.transaction.UserTransaction; import javax.ws.rs.*; import javax.ws.rs.core.*; +import javax.xml.bind.annotation.XmlRootElement; import java.io.*; import java.net.URI; import java.nio.file.Files; @@ -33,6 +35,14 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; +@XmlRootElement +class PkgTuple +{ + String md5sum; + public String filename; + public long size; +} + @CORSManaged @Singleton @Path("/pkg") @@ -41,7 +51,7 @@ import java.util.logging.Logger; @TransactionManagement(TransactionManagementType.BEAN) public class PacmanWebService { - private SortedMap>> cachedMap = null; + private SortedMap>> cachedMap = null; @PersistenceContext(unitName = "jpacrepo_pu") private EntityManager em; @@ -63,6 +73,42 @@ public class PacmanWebService @DefaultConfiguration private ApplicationContext ctx; + + private SortedMap>> getCachedMap() + { + SortedMap>> result; + if (ctx.invalidateCache) + { + result = new TreeMap<>(); + TypedQuery query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName, pkg.size, pkg.md5sum FROM PkgData pkg", Object[].class); + for (Object[] pkg : query.getResultList()) + { + String name = (String) pkg[0]; + String version = (String) pkg[1]; + String arch = (String) pkg[2]; + String filename = (String) pkg[3]; + long size = (long) pkg[4]; + String md5sum = (String) pkg[5]; + result.putIfAbsent(name, new TreeMap<>()); + SortedMap> map = result.get(name); + map.putIfAbsent(version, new TreeMap<>()); + SortedMap map2 = map.get(version); + + PkgTuple tuple = new PkgTuple(); + tuple.filename = filename; + tuple.size = size; + tuple.md5sum = md5sum; + map2.putIfAbsent(arch, tuple); + } + cachedMap = result; + ctx.invalidateCache = false; + } + else + { + result = cachedMap; + } + return result; + } @GET @Path("searchByName/{name}") public Response searchByName(@PathParam("name") String name) @@ -116,33 +162,22 @@ public class PacmanWebService @GET @Produces(MediaType.APPLICATION_JSON) @Path("map") - public Response getPackageMap() + public Response getPackageMap(@Context Request request) { - SortedMap>> result; - if (ctx.invalidateCache) + CacheControl cc = new CacheControl(); + cc.setMaxAge(86400); + cc.setMustRevalidate(true); + cc.setNoCache(true); + SortedMap>> result = getCachedMap(); + EntityTag etag = new EntityTag(Integer.toString(result.hashCode())); + Response.ResponseBuilder builder = request.evaluatePreconditions(etag); + if (builder == null) { - result = new TreeMap<>(); - TypedQuery query = em.createQuery("SELECT pkg.name.id, pkg.version, pkg.arch, pkg.fileName FROM PkgData pkg", Object[].class); - for (Object[] pkg : query.getResultList()) - { - String name = (String) pkg[0]; - String version = (String) pkg[1]; - String arch = (String) pkg[2]; - String filename = (String) pkg[3]; - result.putIfAbsent(name, new TreeMap<>()); - SortedMap> map = result.get(name); - map.putIfAbsent(version, new TreeMap<>()); - SortedMap map2 = map.get(version); - map2.putIfAbsent(arch, filename); - } - cachedMap = result; - ctx.invalidateCache = false; + builder = Response.ok(result); + builder.tag(etag); } - else - { - result = cachedMap; - } - return Response.ok(result).build(); + builder.cacheControl(cc); + return builder.build(); } @GET @@ -179,6 +214,27 @@ public class PacmanWebService return manageQueryResult(fnquery.getResultList(), true); } + @GET + @Path("filesize/{filename}") + public Response getFileSize(@PathParam("filename") String fileName, @Context Request request) + { + CacheControl cc = new CacheControl(); + cc.setMaxAge(86400); + cc.setMustRevalidate(true); + cc.setNoCache(true); + EntityTag etag = new EntityTag(Integer.toString(getCachedMap().hashCode())); + Response.ResponseBuilder builder = request.evaluatePreconditions(etag); + if (builder == null) + { + File res = ctx.getFile(fileName); + if (!res.exists()) throw new NotFoundException(String.format("File '%s' was not found", fileName)); + builder = Response.ok(res.length()); + builder.tag(etag); + } + builder.cacheControl(cc); + return builder.build(); + } + @GET @Path("download/{filename}") @Produces(MediaType.APPLICATION_OCTET_STREAM) @@ -356,7 +412,8 @@ public class PacmanWebService @OPTIONS @Path("/downloadTar") @Produces("text/plain; charset=UTF-8") - public Response options() { + public Response options() + { return Response.ok("POST, OPTIONS").build(); } @@ -367,9 +424,10 @@ public class PacmanWebService public Response downloadTar(@FormParam("pkgs") String formData) { String[] files = formData.split(" "); - for(String fname : files) + for (String fname : files) { - if(!ctx.getFile(fname).exists()) throw new NotFoundException(String.format("Package file '%s' does not exist", fname)); + if (!ctx.getFile(fname).exists()) + throw new NotFoundException(String.format("Package file '%s' does not exist", fname)); } StreamingOutput stream = new StreamingOutput() {