diff --git a/gradle.properties b/gradle.properties index a65f733..7a50a16 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ -jpacrepo.version=2023.07 +jpacrepo.version=2023.10.04 -lys.version=2023.07.28 +lys.version=2023.10.01 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 249e583..033e24c 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 84a0b92..ac72c34 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index a69d9cb..fcb6fca 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +80,10 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +130,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,6 +197,10 @@ if "$cygwin" || "$msys" ; then done fi + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + # Collect all arguments for the java command; # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # shell script including quotes and variable substitutions, so put them in diff --git a/gradlew.bat b/gradlew.bat index 53a6b23..6689b85 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -26,6 +26,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% diff --git a/src/main/java/net/woggioni/jpacrepo/service/PacmanServiceEJB.java b/src/main/java/net/woggioni/jpacrepo/service/PacmanServiceEJB.java index 90ff1c9..3ba13ab 100644 --- a/src/main/java/net/woggioni/jpacrepo/service/PacmanServiceEJB.java +++ b/src/main/java/net/woggioni/jpacrepo/service/PacmanServiceEJB.java @@ -96,74 +96,70 @@ public class PacmanServiceEJB implements PacmanServiceLocal { @TransactionAttribute(TransactionAttributeType.REQUIRED) @Schedule(hour = "4", minute = "00", persistent = false) public void syncDB() { - try { - logger.info("Starting repository cleanup"); - //Removes from DB the packages that have been deleted from filesystem - logger.info("Searching for packages that are no more in the filesystem"); - List resultList = Queries.listAllPackageFiles(em) - .getResultList(); - logger.info("Got list of filenames from db"); - Set knownPkg = resultList.stream().filter(fileName -> { - Path file = ctx.getFile(fileName); - boolean result = Files.exists(file); - if (!result) { - logger.info("Removing package {} which was not found in filesystem", file.getFileName()); - Queries.getPackageByFileName(em, file.getFileName().toString()) - .getResultList() - .forEach(pkgData -> deletePkgData(em, pkgData)); - } - return result; - }).collect(Collectors.toUnmodifiableSet()); - logger.info("Searching for new packages or packages that were modified after being added to the database"); - CompletionService completionService = new ExecutorCompletionService<>(executor); - final Set> inProgress = new HashSet<>(); - final int maxInProgress = Runtime.getRuntime().availableProcessors() * 5; - Sup> fileListStreamSupplier = () -> Files.list(ctx.getRepoFolder()).filter((Path file) -> { - String name = file.getFileName().toString(); - return name.endsWith(".pkg.tar.xz") || name.endsWith(".pkg.tar.zst") || name.endsWith(".pkg.tar.gz"); - }); - long[] count = new long[] {0}; - long totalPackages = fileListStreamSupplier.get().count(); + logger.info("Starting repository cleanup"); + //Removes from DB the packages that have been deleted from filesystem + logger.info("Searching for packages that are no more in the filesystem"); + List resultList = Queries.listAllPackageFiles(em) + .getResultList(); + logger.info("Got list of filenames from db"); + Set knownPkg = resultList.stream().filter(fileName -> { + Path file = ctx.getFile(fileName); + boolean result = Files.exists(file); + if (!result) { + logger.info("Removing package {} which was not found in filesystem", file.getFileName()); + Queries.getPackageByFileName(em, file.getFileName().toString()) + .getResultList() + .forEach(pkgData -> deletePkgData(em, pkgData)); + } + return result; + }).collect(Collectors.toUnmodifiableSet()); + logger.info("Searching for new packages or packages that were modified after being added to the database"); + CompletionService completionService = new ExecutorCompletionService<>(executor); + final Set> inProgress = new HashSet<>(); + final int maxInProgress = Runtime.getRuntime().availableProcessors() * 5; + Sup> fileListStreamSupplier = () -> Files.list(ctx.getRepoFolder()).filter((Path file) -> { + String name = file.getFileName().toString(); + return name.endsWith(".pkg.tar.xz") || name.endsWith(".pkg.tar.zst") || name.endsWith(".pkg.tar.gz"); + }); + long[] count = new long[]{0}; + long totalPackages = fileListStreamSupplier.get().count(); - Con persistPackages = (Boolean drain) -> { - while ((drain && inProgress.size() > 0) || inProgress.size() > maxInProgress) { - Optional.ofNullable(completionService.poll(1, TimeUnit.SECONDS)) - .ifPresent((Con>) future -> { - inProgress.remove(future); - PkgData pkgData; - try { - pkgData = future.get(); - } catch (ExecutionException ee) { - throw ee.getCause(); - } - persistPackage(em, pkgData, ++count[0], totalPackages); - }); - } - }; - fileListStreamSupplier.get().forEach((Con) file -> { - if (!knownPkg.contains(file.getFileName().toString()) || ((Sup) () -> { - Instant result = Queries.getUpdateTimestampByFileName(em, file.getFileName().toString()).getSingleResult(); - return Files.getLastModifiedTime(file).toMillis() > result.toEpochMilli(); - }).get()) { - inProgress.add(completionService.submit(() -> { - try { - return PkgDataImpl.parseFile(file, CompressionFormatImpl.guess(file)); - } catch (Exception ex) { - logger.error(String.format("Error parsing '%s'", file.toAbsolutePath()), ex); - throw ex; - } - })); - } - persistPackages.accept(false); - }); - persistPackages.accept(true); - logger.info("Removing obsolete packages"); - deleteOld(em); - logger.info("Repository cleanup completed successfully"); - packageCache.invalidateCache(); - } finally { - em.close(); - } + Con persistPackages = (Boolean drain) -> { + while ((drain && inProgress.size() > 0) || inProgress.size() > maxInProgress) { + Optional.ofNullable(completionService.poll(1, TimeUnit.SECONDS)) + .ifPresent((Con>) future -> { + inProgress.remove(future); + PkgData pkgData; + try { + pkgData = future.get(); + } catch (ExecutionException ee) { + throw ee.getCause(); + } + persistPackage(em, pkgData, ++count[0], totalPackages); + }); + } + }; + fileListStreamSupplier.get().forEach((Con) file -> { + if (!knownPkg.contains(file.getFileName().toString()) || ((Sup) () -> { + Instant result = Queries.getUpdateTimestampByFileName(em, file.getFileName().toString()).getSingleResult(); + return Files.getLastModifiedTime(file).toMillis() > result.toEpochMilli(); + }).get()) { + inProgress.add(completionService.submit(() -> { + try { + return PkgDataImpl.parseFile(file, CompressionFormatImpl.guess(file)); + } catch (Exception ex) { + logger.error(String.format("Error parsing '%s'", file.toAbsolutePath()), ex); + throw ex; + } + })); + } + persistPackages.accept(false); + }); + persistPackages.accept(true); + logger.info("Removing obsolete packages"); + deleteOld(em); + logger.info("Repository cleanup completed successfully"); + packageCache.invalidateCache(); } private void persistPackage(EntityManager em, PkgData pkgData, long count, long totalPackages) { @@ -194,6 +190,7 @@ public class PacmanServiceEJB implements PacmanServiceLocal { em.remove(pkg); Files.delete(ctx.getFile(pkg)); } + private void deleteOld(EntityManager em) { Instant cutoff = Instant.now().minus(365 * 3, ChronoUnit.DAYS); Queries.getOldPackages2Delete(em, cutoff, 3L) @@ -266,7 +263,7 @@ public class PacmanServiceEJB implements PacmanServiceLocal { public Set missingFiles(Collection fileNames) { Stream result = fileNames.stream(); Set existing = Queries.getExistingFiles(em, fileNames) - .getResultStream().collect(CollectionUtils.toUnmodifiableTreeSet()); + .getResultStream().collect(CollectionUtils.toUnmodifiableTreeSet()); return result.filter(JWO.not(existing::contains)) .collect(CollectionUtils.toUnmodifiableTreeSet()); } @@ -281,18 +278,19 @@ public class PacmanServiceEJB implements PacmanServiceLocal { PkgId pkgId = tuple.get(0, PkgId.class); String filename = tuple.get(1, String.class); long size = tuple.get(2, Long.class); - String md5sum = tuple.get(3, String.class);; + String md5sum = tuple.get(3, String.class); + ; PkgTuple pkgTuple = new PkgTuple(); pkgTuple.setFileName(filename); pkgTuple.setSize(size); pkgTuple.setMd5sum(md5sum); return Tuple2.newInstance(pkgId, pkgTuple); }).collect( - CollectionUtils.toUnmodifiableTreeMap( - Tuple2::get_1, - Tuple2::get_2, - PkgIdComparator.getComparator() - ) + CollectionUtils.toUnmodifiableTreeMap( + Tuple2::get_1, + Tuple2::get_2, + PkgIdComparator.getComparator() + ) ); } @@ -308,12 +306,12 @@ public class PacmanServiceEJB implements PacmanServiceLocal { List savedFiles = searchByFileName(fileName); if (savedFiles.size() > 0) return false; else { - try(OutputStream output = Files.newOutputStream(file)) { + try (OutputStream output = Files.newOutputStream(file)) { JWO.copy(input, output, 0x10000); PkgData pkg = PkgDataImpl.parseFile(file, - CompressionFormatImpl.guess(Paths.get(fileName))); + CompressionFormatImpl.guess(Paths.get(fileName))); pkg.setFileName(fileName); - Optional.ofNullable(em.find(PkgData.class, pkg.getId())).ifPresent((Con) (pkgData -> { + Optional.ofNullable(em.find(PkgData.class, pkg.getId())).ifPresent((Con) (pkgData -> { em.remove(pkgData); Files.delete(ctx.getRepoFolder().resolve(pkgData.getFileName())); })); @@ -324,7 +322,7 @@ public class PacmanServiceEJB implements PacmanServiceLocal { return true; } catch (Throwable t) { Files.delete(file); - throw t; + throw t; } } }