Skip to content

Commit 22f8ed0

Browse files
authored
Fix OOM when updating very large metadata datasources
1 parent c9920c8 commit 22f8ed0

3 files changed

Lines changed: 346 additions & 35 deletions

File tree

‎cloudinary-core/src/main/java/com/cloudinary/Api.java‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -755,12 +755,28 @@ public ApiResponse updateMetadataField(String fieldExternalId, MetadataField fie
755755
* @param fieldExternalId The id of the field to update
756756
* @param entries A list of datasource entries. Existing entries (according to entry id) will be updated,
757757
* new entries will be added.
758-
* @return The updated field definition.
758+
* @return The datasource entries (a map with a {@code values} key).
759759
* @throws Exception
760760
*/
761761
public ApiResponse updateMetadataFieldDatasource(String fieldExternalId, List<MetadataDataSource.Entry> entries) throws Exception {
762+
return updateMetadataFieldDatasource(fieldExternalId, entries, null);
763+
}
764+
765+
/**
766+
* Update a data source's entries (see {@link #updateMetadataFieldDatasource(String, List)}), with extra options.
767+
*
768+
* @param fieldExternalId The id of the field to update
769+
* @param entries A list of datasource entries. Existing entries (according to entry id) will be updated,
770+
* new entries will be added.
771+
* @param options Extra options. Set {@code skip_response_body} to {@code true} to skip parsing the response
772+
* (which echoes the whole datasource) and return an empty response; recommended for very large
773+
* datasources to avoid excessive memory usage.
774+
* @return The datasource entries (a map with a {@code values} key), or an empty response if {@code skip_response_body} is set.
775+
* @throws Exception
776+
*/
777+
public ApiResponse updateMetadataFieldDatasource(String fieldExternalId, List<MetadataDataSource.Entry> entries, Map options) throws Exception {
762778
List<String> uri = Arrays.asList("metadata_fields", fieldExternalId, "datasource");
763-
return callApi(HttpMethod.PUT, uri, Collections.singletonMap("values", entries), Collections.singletonMap("content_type", "json"));
779+
return callApi(HttpMethod.PUT, uri, Collections.singletonMap("values", entries), withJsonContentType(options));
764780
}
765781

766782
/**
@@ -772,8 +788,23 @@ public ApiResponse updateMetadataFieldDatasource(String fieldExternalId, List<Me
772788
* @throws Exception
773789
*/
774790
public ApiResponse deleteDatasourceEntries(String fieldExternalId, List<String> entriesExternalId) throws Exception {
791+
return deleteDatasourceEntries(fieldExternalId, entriesExternalId, null);
792+
}
793+
794+
/**
795+
* Delete data source entries for a given field, with extra options.
796+
*
797+
* @param fieldExternalId The id of the field to update
798+
* @param entriesExternalId The ids of all the entries to delete from the data source
799+
* @param options Extra options. Set {@code skip_response_body} to {@code true} to skip parsing the response
800+
* (which echoes the whole datasource) and return an empty response; recommended for very large
801+
* datasources to avoid excessive memory usage.
802+
* @return The remaining datasource entries, or an empty response if {@code skip_response_body} is set.
803+
* @throws Exception
804+
*/
805+
public ApiResponse deleteDatasourceEntries(String fieldExternalId, List<String> entriesExternalId, Map options) throws Exception {
775806
List<String> uri = Arrays.asList("metadata_fields", fieldExternalId, "datasource");
776-
return callApi(HttpMethod.DELETE, uri, Collections.singletonMap("external_ids", entriesExternalId), Collections.emptyMap());
807+
return callApi(HttpMethod.DELETE, uri, Collections.singletonMap("external_ids", entriesExternalId), withJsonContentType(options));
777808
}
778809

779810
/**
@@ -785,8 +816,31 @@ public ApiResponse deleteDatasourceEntries(String fieldExternalId, List<String>
785816
* @throws Exception
786817
*/
787818
public ApiResponse restoreDatasourceEntries(String fieldExternalId, List<String> entriesExternalId) throws Exception {
819+
return restoreDatasourceEntries(fieldExternalId, entriesExternalId, null);
820+
}
821+
822+
/**
823+
* Restore deleted data source entries for a given field, with extra options.
824+
*
825+
* @param fieldExternalId The id of the field to operate
826+
* @param entriesExternalId The ids of all the entries to restore from the data source
827+
* @param options Extra options. Set {@code skip_response_body} to {@code true} to skip parsing the response.
828+
* @return The datasource entries state after restore, or an empty response if {@code skip_response_body} is set.
829+
* @throws Exception
830+
*/
831+
public ApiResponse restoreDatasourceEntries(String fieldExternalId, List<String> entriesExternalId, Map options) throws Exception {
788832
List<String> uri = Arrays.asList("metadata_fields", fieldExternalId, "datasource_restore");
789-
return callApi(HttpMethod.POST, uri, Collections.singletonMap("external_ids", entriesExternalId), Collections.singletonMap("content_type", "json"));
833+
return callApi(HttpMethod.POST, uri, Collections.singletonMap("external_ids", entriesExternalId), withJsonContentType(options));
834+
}
835+
836+
@SuppressWarnings({"rawtypes", "unchecked"})
837+
private static Map withJsonContentType(Map options) {
838+
Map merged = new java.util.HashMap();
839+
if (options != null) {
840+
merged.putAll(options);
841+
}
842+
merged.put("content_type", "json");
843+
return merged;
790844
}
791845

792846
/**

‎cloudinary-http5/src/main/java/com/cloudinary/http5/ApiStrategy.java‎

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,15 @@
2424
import org.apache.hc.core5.util.Timeout;
2525
import org.cloudinary.json.JSONException;
2626
import org.cloudinary.json.JSONObject;
27+
import org.cloudinary.json.JSONTokener;
2728

2829
import java.io.IOException;
30+
import java.io.InputStream;
31+
import java.io.InputStreamReader;
2932
import java.lang.reflect.Constructor;
3033
import java.net.URISyntaxException;
3134
import java.nio.charset.StandardCharsets;
35+
import java.util.Collections;
3236
import java.util.List;
3337
import java.util.Map;
3438

@@ -83,55 +87,72 @@ public ApiResponse callApi(Api.HttpMethod method, String apiUrl, Map<String, ?>
8387

8488
request.setHeader("Authorization", autorizationHeader);
8589

86-
return getApiResponse(request);
90+
return getApiResponse(request, options);
8791
}
8892

89-
private ApiResponse getApiResponse(HttpUriRequestBase request) throws Exception {
90-
String responseData = null;
91-
int code = 0;
92-
CloseableHttpResponse response;
93-
try {
94-
response = client.execute(request);
95-
code = response.getCode();
93+
@SuppressWarnings({"rawtypes", "unchecked"})
94+
private ApiResponse getApiResponse(HttpUriRequestBase request, Map options) throws Exception {
95+
boolean skipResponseBody = options != null && ObjectUtils.asBoolean(options.get("skip_response_body"), false);
96+
97+
try (CloseableHttpResponse response = client.execute(request)) {
98+
int code = response.getCode();
9699
HttpEntity entity = response.getEntity();
97-
if (entity != null) {
98-
responseData = EntityUtils.toString(entity, StandardCharsets.UTF_8);
100+
101+
if (code != 200) {
102+
String responseData = entity != null ? EntityUtils.toString(entity, StandardCharsets.UTF_8) : null;
103+
throw buildError(code, responseData);
104+
}
105+
106+
if (skipResponseBody) {
107+
EntityUtils.consumeQuietly(entity);
108+
return new Response(response, Collections.<String, Object>emptyMap());
109+
}
110+
111+
if (entity == null) {
112+
throw new RuntimeException("Invalid JSON response from server: empty response body");
99113
}
100-
} catch (IOException e) {
101-
throw new GeneralError("Error executing request: " + e.getMessage());
102-
}
103114

104-
if (code != 200) {
115+
// Parse straight from the stream (explicit UTF-8) to avoid buffering the whole body as a String.
105116
Map<String, Object> result;
106-
try {
107-
JSONObject responseJSON = new JSONObject(responseData);
117+
try (InputStream content = entity.getContent()) {
118+
JSONObject responseJSON = new JSONObject(new JSONTokener(new InputStreamReader(content, StandardCharsets.UTF_8)));
108119
result = ObjectUtils.toMap(responseJSON);
109120
} catch (JSONException e) {
121+
if (e.getCause() instanceof IOException) {
122+
// network failure while reading the body: same error type as before streaming the parse
123+
throw new GeneralError("Error executing request: " + e.getMessage());
124+
}
110125
throw new RuntimeException("Invalid JSON response from server " + e.getMessage());
111126
}
127+
EntityUtils.consumeQuietly(entity);
112128

113-
// Extract the error message from the result map
114-
String message = (String) ((Map<String, Object>) result.get("error")).get("message");
115-
116-
// Get the appropriate exception class based on status code
117-
Class<? extends Exception> exceptionClass = Api.CLOUDINARY_API_ERROR_CLASSES.get(code);
118-
if (exceptionClass != null) {
119-
Constructor<? extends Exception> exceptionConstructor = exceptionClass.getConstructor(String.class);
120-
throw exceptionConstructor.newInstance(message);
121-
} else {
122-
throw new GeneralError("Server returned unexpected status code - " + code + " - " + responseData);
123-
}
129+
return new Response(response, result);
130+
} catch (IOException e) {
131+
throw new GeneralError("Error executing request: " + e.getMessage());
124132
}
133+
}
125134

135+
@SuppressWarnings("unchecked")
136+
private Exception buildError(int code, String responseData) throws Exception {
126137
Map<String, Object> result;
138+
if (responseData == null) {
139+
throw new RuntimeException("Invalid JSON response from server null");
140+
}
127141
try {
128-
JSONObject responseJSON = new JSONObject(responseData);
129-
result = ObjectUtils.toMap(responseJSON);
142+
result = ObjectUtils.toMap(new JSONObject(responseData));
130143
} catch (JSONException e) {
131144
throw new RuntimeException("Invalid JSON response from server " + e.getMessage());
132145
}
133146

134-
return new Response(response, result);
147+
Object error = result.get("error");
148+
String message = error instanceof Map ? (String) ((Map<String, Object>) error).get("message") : null;
149+
150+
Class<? extends Exception> exceptionClass = Api.CLOUDINARY_API_ERROR_CLASSES.get(code);
151+
if (exceptionClass != null) {
152+
Constructor<? extends Exception> exceptionConstructor = exceptionClass.getConstructor(String.class);
153+
return exceptionConstructor.newInstance(message);
154+
}
155+
return new GeneralError("Server returned unexpected status code - " + code + " - " + responseData);
135156
}
136157

137158
@Override
@@ -144,7 +165,7 @@ public ApiResponse callAccountApi(Api.HttpMethod method, String apiUrl, Map<Stri
144165
request.setHeader("Authorization", authorizationHeader);
145166

146167
// Execute the request and return the response
147-
return getApiResponse(request);
168+
return getApiResponse(request, options);
148169
}
149170

150171
private HttpUriRequestBase prepareRequest(Api.HttpMethod method, String apiUrl, Map<String, ? extends Object> params, Map<String, ?> options) throws URISyntaxException {

0 commit comments

Comments
 (0)