Report hashes of jars that we can resolve to dependencies

agent_api/src/main/java/dev/aikido/agent_api/background/HeartbeatTask.java

@@ -44,16 +44,18 @@ public void run() {
         RouteEntry[] routes = RoutesStore.getRoutesAsList();
         List<User> users = UsersStore.getUsersAsList();
         List<RuntimePackage> packages = RuntimePackagesStore.getPackagesAsList();
+        List<JavaArtifact> javaArtifacts = JavaArtifactsStore.getArtifactsAsList();
 
         // Clear data :
         StatisticsStore.clear();
         HostnamesStore.clear();
         RoutesStore.clear();
         UsersStore.clear();
         RuntimePackagesStore.clear();
+        JavaArtifactsStore.clear();
 
         // Create and send event :
-        Heartbeat.HeartbeatEvent event = Heartbeat.get(stats, hostnames, routes, users, packages);
+        Heartbeat.HeartbeatEvent event = Heartbeat.get(stats, hostnames, routes, users, packages, javaArtifacts);
         Optional<APIResponse> res = api.report(event);
         res.ifPresent(ServiceConfigStore::updateFromAPIResponse);
     }

agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/events/Heartbeat.java

@@ -2,6 +2,7 @@
 
 import dev.aikido.agent_api.background.cloud.GetManagerInfo;
 import dev.aikido.agent_api.storage.Hostnames;
+import dev.aikido.agent_api.storage.JavaArtifact;
 import dev.aikido.agent_api.storage.RuntimePackage;
 import dev.aikido.agent_api.storage.ServiceConfigStore;
 import dev.aikido.agent_api.storage.statistics.Statistics;
@@ -24,16 +25,20 @@ public record HeartbeatEvent(
             RouteEntry[] routes,
             List<User> users,
             List<RuntimePackage> packages,
+            List<JavaArtifact> javaArtifacts,
             boolean middlewareInstalled
     ) implements APIEvent {}
 
     public static HeartbeatEvent get(
             Statistics.StatsRecord stats, Hostnames.HostnameEntry[] hostnames, RouteEntry[] routes,
-            List<User> users, List<RuntimePackage> packages
+            List<User> users, List<RuntimePackage> packages, List<JavaArtifact> javaArtifacts
     ) {
         long time = getUnixTimeMS(); // Get current time
         GetManagerInfo.ManagerInfo agent = getManagerInfo();
         boolean middlewareInstalled = ServiceConfigStore.getConfig().isMiddlewareInstalled();
-        return new HeartbeatEvent("heartbeat", agent, time, stats, hostnames, routes, users, packages, middlewareInstalled);
+        return new HeartbeatEvent(
+                "heartbeat", agent, time, stats, hostnames, routes, users,
+                packages, javaArtifacts, middlewareInstalled
+        );
     }
 }

agent_api/src/main/java/dev/aikido/agent_api/helpers/Hashing.java

@@ -0,0 +1,34 @@
+package dev.aikido.agent_api.helpers;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public final class Hashing {
+    private static final char[] HEX = "0123456789abcdef".toCharArray();
+
+    private Hashing() {}
+
+    public static String sha1(InputStream input) throws IOException {
+        final MessageDigest digest;
+        try {
+            digest = MessageDigest.getInstance("SHA-1");
+        } catch (NoSuchAlgorithmException impossible) {
+            throw new IllegalStateException("SHA-1 is not available", impossible);
+        }
+        byte[] buffer = new byte[8192];
+        int read;
+        while ((read = input.read(buffer)) != -1) {
+            digest.update(buffer, 0, read);
+        }
+        byte[] hash = digest.digest();
+        char[] encoded = new char[hash.length * 2];
+        for (int i = 0; i < hash.length; i++) {
+            int value = hash[i] & 0xff;
+            encoded[i * 2] = HEX[value >>> 4];
+            encoded[i * 2 + 1] = HEX[value & 0x0f];
+        }
+        return new String(encoded);
+    }
+}

agent_api/src/main/java/dev/aikido/agent_api/helpers/packages/JarPackageScanner.java

@@ -1,5 +1,6 @@
 package dev.aikido.agent_api.helpers.packages;
 
+import dev.aikido.agent_api.helpers.Hashing;
 import dev.aikido.agent_api.storage.RuntimePackage;
 
 import java.io.BufferedInputStream;
@@ -29,28 +30,59 @@ private JarPackageScanner() {}
     public static List<RuntimePackage> findMavenPackages(
             String classResourceUrl,
             long requiredAt
+    ) {
+        return scan(classResourceUrl, requiredAt).packages();
+    }
+
+    public static JarScanResult scan(
+            String classResourceUrl,
+            long requiredAt
     ) {
         try {
             JarLocation location = JarLocation.parse(classResourceUrl);
             if (location == null) {
-                return List.of();
+                return JarScanResult.empty();
             }
             if (location.nestedEntry() == null) {
+                List<RuntimePackage> packages;
                 try (InputStream input = new BufferedInputStream(Files.newInputStream(location.outerJar()))) {
-                    return findMavenPackages(input, requiredAt);
+                    packages = findMavenPackages(input, requiredAt);
                 }
+                return result(location.outerJar(), packages);
             }
             try (JarFile outerJar = new JarFile(location.outerJar().toFile())) {
                 JarEntry nestedJar = outerJar.getJarEntry(location.nestedEntry());
                 if (nestedJar == null) {
-                    return List.of();
+                    return JarScanResult.empty();
+                }
+                List<RuntimePackage> packages;
+                try (InputStream input = new BufferedInputStream(outerJar.getInputStream(nestedJar))) {
+                    packages = findMavenPackages(input, requiredAt);
+                }
+                if (!packages.isEmpty()) {
+                    return new JarScanResult(packages, null);
                 }
                 try (InputStream input = new BufferedInputStream(outerJar.getInputStream(nestedJar))) {
-                    return findMavenPackages(input, requiredAt);
+                    return new JarScanResult(packages, Hashing.sha1(input));
                 }
             }
         } catch (IOException | RuntimeException ignored) {
-            return List.of();
+            return JarScanResult.empty();
+        }
+    }
+
+    private static JarScanResult result(Path jar, List<RuntimePackage> packages) throws IOException {
+        if (!packages.isEmpty()) {
+            return new JarScanResult(packages, null);
+        }
+        try (InputStream input = new BufferedInputStream(Files.newInputStream(jar))) {
+            return new JarScanResult(packages, Hashing.sha1(input));
+        }
+    }
+
+    public record JarScanResult(List<RuntimePackage> packages, String sha1) {
+        private static JarScanResult empty() {
+            return new JarScanResult(List.of(), null);
         }
     }
 

agent_api/src/main/java/dev/aikido/agent_api/helpers/packages/RuntimePackageCollector.java

@@ -1,5 +1,7 @@
 package dev.aikido.agent_api.helpers.packages;
 
+import dev.aikido.agent_api.storage.JavaArtifact;
+import dev.aikido.agent_api.storage.JavaArtifactsStore;
 import dev.aikido.agent_api.storage.RuntimePackagesStore;
 
 import java.security.ProtectionDomain;
@@ -45,7 +47,11 @@ private static void processLocations() {
         while (!Thread.currentThread().isInterrupted()) {
             try {
                 ObservedLocation location = PENDING_LOCATIONS.take();
-                RuntimePackagesStore.addAll(JarPackageScanner.findMavenPackages(location.url(), location.requiredAt()));
+                JarPackageScanner.JarScanResult result = JarPackageScanner.scan(location.url(), location.requiredAt());
+                RuntimePackagesStore.addAll(result.packages());
+                if (result.sha1() != null) {
+                    JavaArtifactsStore.add(new JavaArtifact(result.sha1(), location.requiredAt()));
+                }
             } catch (InterruptedException interrupted) {
                 Thread.currentThread().interrupt();
             } catch (Throwable ignored) {

agent_api/src/main/java/dev/aikido/agent_api/storage/JavaArtifact.java

@@ -0,0 +1,3 @@
+package dev.aikido.agent_api.storage;
+
+public record JavaArtifact(String sha1, long requiredAt) {}

agent_api/src/main/java/dev/aikido/agent_api/storage/JavaArtifactsStore.java

@@ -0,0 +1,26 @@
+package dev.aikido.agent_api.storage;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public final class JavaArtifactsStore {
+    private static final ConcurrentMap<String, JavaArtifact> ARTIFACTS = new ConcurrentHashMap<>();
+
+    private JavaArtifactsStore() {}
+
+    public static void add(JavaArtifact artifact) {
+        ARTIFACTS.putIfAbsent(artifact.sha1(), artifact);
+    }
+
+    public static List<JavaArtifact> getArtifactsAsList() {
+        return ARTIFACTS.values().stream()
+                .sorted(Comparator.comparing(JavaArtifact::sha1))
+                .toList();
+    }
+
+    public static void clear() {
+        ARTIFACTS.clear();
+    }
+}

agent_api/src/test/java/background/cloud/api/HeartbeatEventTest.java

@@ -4,6 +4,7 @@
 import dev.aikido.agent_api.background.cloud.GetManagerInfo;
 import dev.aikido.agent_api.background.cloud.api.events.Heartbeat;
 import dev.aikido.agent_api.storage.Hostnames;
+import dev.aikido.agent_api.storage.JavaArtifact;
 import dev.aikido.agent_api.storage.statistics.Statistics;
 import dev.aikido.agent_api.storage.routes.RouteEntry;
 import dev.aikido.agent_api.context.User;
@@ -50,12 +51,15 @@ public void testGetHeartbeatEvent() {
         hostnames.add("aikido.dev", 8080);
         RouteEntry[] routes = new RouteEntry[0]; // Replace with actual RouteEntry array if needed
         List<User> users = Collections.emptyList(); // Replace with actual User list if needed
+        List<JavaArtifact> javaArtifacts = List.of(new JavaArtifact("0123456789012345678901234567890123456789", 123L));
 
         mockedGetManagerInfo.when(GetManagerInfo::getManagerInfo).thenReturn(managerInfo);
         ServiceConfigStore.setMiddlewareInstalled(false);
 
         // Act
-        Heartbeat.HeartbeatEvent event = Heartbeat.get(stats, hostnames.asArray(), routes, users, Collections.emptyList());
+        Heartbeat.HeartbeatEvent event = Heartbeat.get(
+                stats, hostnames.asArray(), routes, users, Collections.emptyList(), javaArtifacts
+        );
 
         // Assert
         assertEquals("heartbeat", event.type());
@@ -64,11 +68,14 @@ public void testGetHeartbeatEvent() {
         assertArrayEquals(hostnames.asArray(), event.hostnames());
         assertEquals(routes, event.routes());
         assertEquals(users, event.users());
+        assertEquals(javaArtifacts, event.javaArtifacts());
         assertFalse(event.middlewareInstalled());
 
         // Test middleware installed as well :
         ServiceConfigStore.setMiddlewareInstalled(true);
-        Heartbeat.HeartbeatEvent event2 = Heartbeat.get(stats, hostnames.asArray(), routes, users, Collections.emptyList());
+        Heartbeat.HeartbeatEvent event2 = Heartbeat.get(
+                stats, hostnames.asArray(), routes, users, Collections.emptyList(), javaArtifacts
+        );
         assertTrue(event2.middlewareInstalled());
     }
 }

agent_api/src/test/java/helpers/packages/JarPackageScannerTest.java

@@ -8,15 +8,53 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.MessageDigest;
 import java.util.List;
 import java.util.jar.Attributes;
 import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 class JarPackageScannerTest {
+    @Test
+    void hashesJarWithoutMavenCoordinates() throws Exception {
+        Path jar = Files.createTempFile("aikido-package", ".jar");
+        try (JarOutputStream output = new JarOutputStream(Files.newOutputStream(jar))) {
+            output.putNextEntry(new JarEntry("org/example/Demo.class"));
+            output.write(new byte[] {1, 2, 3});
+        }
+
+        JarPackageScanner.JarScanResult result = JarPackageScanner.scan(jar.toUri().toString(), 123L);
+
+        assertEquals(List.of(), result.packages());
+        byte[] hash = MessageDigest.getInstance("SHA-1").digest(Files.readAllBytes(jar));
+        assertEquals(toHex(hash), result.sha1());
+        Files.deleteIfExists(jar);
+    }
+
+    @Test
+    void readsAllMavenCoordinatesFromFlattenedUberJarWithoutHashingIt() throws IOException {
+        Path jar = Files.createTempFile("aikido-uber-package", ".jar");
+        try (JarOutputStream output = new JarOutputStream(Files.newOutputStream(jar))) {
+            output.putNextEntry(new JarEntry("META-INF/maven/org.example/first/pom.properties"));
+            output.write("groupId=org.example\nartifactId=first\nversion=1.0.0\n".getBytes(StandardCharsets.UTF_8));
+            output.putNextEntry(new JarEntry("META-INF/maven/org.example/second/pom.properties"));
+            output.write("groupId=org.example\nartifactId=second\nversion=2.0.0\n".getBytes(StandardCharsets.UTF_8));
+        }
+
+        JarPackageScanner.JarScanResult result = JarPackageScanner.scan(jar.toUri().toString(), 123L);
+
+        assertEquals(List.of(
+                new RuntimePackage("org.example:first", "1.0.0", 123L),
+                new RuntimePackage("org.example:second", "2.0.0", 123L)
+        ), result.packages());
+        assertNull(result.sha1());
+        Files.deleteIfExists(jar);
+    }
+
     @Test
     void readsMavenCoordinates() throws IOException {
         Path jar = Files.createTempFile("aikido-package", ".jar");
@@ -99,4 +137,12 @@ void ignoresUnresolvedMavenProperties() throws IOException {
         assertEquals(List.of(), packages);
         Files.deleteIfExists(jar);
     }
+
+    private static String toHex(byte[] bytes) {
+        StringBuilder result = new StringBuilder(bytes.length * 2);
+        for (byte value : bytes) {
+            result.append(String.format("%02x", value));
+        }
+        return result.toString();
+    }
 }