fix: reduce batch size if API returns 413 (#130)

PostHog · May 8, 2024 · b779a84 · b779a84
1 parent 110b8b4
commit b779a84
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 ## Next
 
+## 3.2.1 - 2024-05-08
+
+- fix: reduce batch size if API returns 413 ([#130](https://github.com/PostHog/posthog-android/pull/130))
+  - `screenshot` image is now a JPEG at 30% quality to reduce size
+
 ## 3.2.0 - 2024-04-30
 
 - recording: add `screenshot` option for session replay instead of wireframe ([#127](https://github.com/PostHog/posthog-android/pull/127))

diff --git a/posthog-android/lint-baseline.xml b/posthog-android/lint-baseline.xml
@@ -25,7 +25,7 @@
 
     <issue
         id="GradleDependency"
-        message="A newer version of androidx.core:core than 1.5.0 is available: 1.13.0"
+        message="A newer version of androidx.core:core than 1.5.0 is available: 1.13.1"
         errorLine1="    implementation(&quot;androidx.core:core:${PosthogBuildConfig.Dependencies.ANDROIDX_CORE}&quot;)"
         errorLine2="                   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
         <location

diff --git a/posthog-android/src/main/java/com/posthog/android/replay/PostHogReplayIntegration.kt b/posthog-android/src/main/java/com/posthog/android/replay/PostHogReplayIntegration.kt
@@ -805,9 +805,11 @@ public class PostHogReplayIntegration(
         }
 
         ByteArrayOutputStream(allocationByteCount).use {
-            compress(Bitmap.CompressFormat.PNG, 30, it)
+            // we can make format and type configurable
+            compress(Bitmap.CompressFormat.JPEG, 30, it)
             val byteArray = it.toByteArray()
-            return Base64.encodeToString(byteArray, Base64.DEFAULT)
+            val encoded = Base64.encodeToString(byteArray, Base64.DEFAULT) ?: return null
+            return "data:image/jpeg;base64,$encoded"
         }
     }
 

diff --git a/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt b/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt
@@ -12,6 +12,7 @@ import java.util.UUID
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.atomic.AtomicBoolean
 import kotlin.concurrent.schedule
+import kotlin.math.max
 import kotlin.math.min
 
 /**
@@ -189,9 +190,8 @@ internal class PostHogQueue(
                 }
             }
         } catch (e: PostHogApiError) {
-            if (e.statusCode < 400) {
-                deleteFiles = false
-            }
+            deleteFiles = deleteFilesIfAPIError(e, config)
+
             throw e
         } catch (e: IOException) {
             // no connection should try again
@@ -315,3 +315,26 @@ internal class PostHogQueue(
             return tempFiles
         }
 }
+
+private fun calcFloor(currentValue: Int): Int {
+    return max(currentValue.floorDiv(2), 1)
+}
+
+internal fun deleteFilesIfAPIError(
+    e: PostHogApiError,
+    config: PostHogConfig,
+): Boolean {
+    if (e.statusCode < 400) {
+        return false
+    }
+    // workaround due to png images exceed our max. limit in kafka
+    if (e.statusCode == 413 && config.maxBatchSize > 1) {
+        // try to reduce the batch size and flushAt until its 1
+        // and if it still throws 413 in the next retry, delete the files since we cannot handle anyway
+        config.maxBatchSize = calcFloor(config.maxBatchSize)
+        config.flushAt = calcFloor(config.flushAt)
+
+        return false
+    }
+    return true
+}
diff --git a/posthog/src/main/java/com/posthog/internal/PostHogSendCachedEventsIntegration.kt b/posthog/src/main/java/com/posthog/internal/PostHogSendCachedEventsIntegration.kt
@@ -186,9 +186,8 @@ internal class PostHogSendCachedEventsIntegration(
                                 PostHogApiEndpoint.SNAPSHOT -> api.snapshot(events)
                             }
                         } catch (e: PostHogApiError) {
-                            if (e.statusCode < 400) {
-                                deleteFiles = false
-                            }
+                            deleteFiles = deleteFilesIfAPIError(e, config)
+
                             throw e
                         } catch (e: IOException) {
                             // no connection should try again

diff --git a/posthog/src/test/java/com/posthog/internal/PostHogQueueTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogQueueTest.kt
@@ -9,6 +9,7 @@ import com.posthog.mockHttp
 import com.posthog.shutdownAndAwaitTermination
 import okhttp3.mockwebserver.MockResponse
 import okhttp3.mockwebserver.SocketPolicy
+import org.junit.Assert.assertFalse
 import org.junit.Rule
 import org.junit.rules.TemporaryFolder
 import java.io.File
@@ -330,4 +331,36 @@ internal class PostHogQueueTest {
         assertEquals(0, File(path, API_KEY).listFiles()!!.size)
         assertEquals(4, http.requestCount)
     }
+
+    @Test
+    fun `reduces batch size if 413`() {
+        val e = PostHogApiError(413, "", null)
+        val config = PostHogConfig(API_KEY)
+
+        assertFalse(deleteFilesIfAPIError(e, config))
+        assertEquals(config.maxBatchSize, 25) // default 50
+        assertEquals(config.flushAt, 10) // default 20
+    }
+
+    @Test
+    fun `delete files if batch is min already`() {
+        val e = PostHogApiError(413, "", null)
+        val config =
+            PostHogConfig(API_KEY).apply {
+                maxBatchSize = 1
+                flushAt = 1
+            }
+
+        assertTrue(deleteFilesIfAPIError(e, config))
+        assertEquals(config.maxBatchSize, 1)
+        assertEquals(config.flushAt, 1)
+    }
+
+    @Test
+    fun `delete files if errored`() {
+        val e = PostHogApiError(400, "", null)
+        val config = PostHogConfig(API_KEY)
+
+        assertTrue(deleteFilesIfAPIError(e, config))
+    }
 }