diff --git a/component-api/src/main/java/org/talend/sdk/component/api/exception/DiscoverSchemaException.java b/component-api/src/main/java/org/talend/sdk/component/api/exception/DiscoverSchemaException.java index a913307ac4c10..f0f6693249ed9 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/exception/DiscoverSchemaException.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/exception/DiscoverSchemaException.java @@ -18,8 +18,11 @@ import javax.json.bind.annotation.JsonbCreator; import javax.json.bind.annotation.JsonbPropertyOrder; -import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; /** * This class is dedicated to Studio's guess schema feature. @@ -29,7 +32,10 @@ * * See me TCOMP-2342 for more details. */ -@Data +@Setter +@Getter +@ToString +@NoArgsConstructor @EqualsAndHashCode(callSuper = true) @JsonbPropertyOrder({ "localizedMessage", "message", "stackTrace", "suppressed", "possibleHandleErrorWith" }) public class DiscoverSchemaException extends RuntimeException { @@ -58,7 +64,7 @@ public enum HandleErrorWith { * This won't query for any user input. * When specifying this option, developer should be sure that no side effect can be generated by connector. */ - EXECUTE_LIFECYCLE; + EXECUTE_LIFECYCLE } private HandleErrorWith possibleHandleErrorWith = HandleErrorWith.EXCEPTION; @@ -69,12 +75,12 @@ public DiscoverSchemaException(final ComponentException e) { public DiscoverSchemaException(final ComponentException e, final HandleErrorWith handling) { super(e.getOriginalMessage(), e.getCause()); - setPossibleHandleErrorWith(handling); + this.possibleHandleErrorWith = handling; } public DiscoverSchemaException(final String message, final HandleErrorWith handling) { super(message); - setPossibleHandleErrorWith(handling); + this.possibleHandleErrorWith = handling; } @JsonbCreator @@ -82,7 +88,6 @@ public DiscoverSchemaException(final String message, final StackTraceElement[] s final HandleErrorWith handling) { super(message); setStackTrace(stackTrace); - setPossibleHandleErrorWith(handling); + this.possibleHandleErrorWith = handling; } - } diff --git a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java index 795801b3f6111..45a7caf1e93c1 100644 --- a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java +++ b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java @@ -28,5 +28,11 @@ public class ErrorPayload { private ErrorDictionary code; + private String subCode; + private String description; + + public ErrorPayload(final ErrorDictionary code, final String description) { + this(code, null, description); + } } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java index b49d797e7a1d9..d30e0a62b217b 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java @@ -42,6 +42,8 @@ import javax.ws.rs.core.Response; import org.talend.sdk.component.api.exception.ComponentException; +import org.talend.sdk.component.api.exception.DiscoverSchemaException; +import org.talend.sdk.component.api.exception.DiscoverSchemaException.HandleErrorWith; import org.talend.sdk.component.runtime.manager.ComponentManager; import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry; import org.talend.sdk.component.runtime.manager.ServiceMeta; @@ -176,16 +178,15 @@ private CompletableFuture doExecuteLocalAction(final String family, fi // check org.talend.sdk.component.server.service.ComponentManagerService.readCurrentLocale if you change it }, Runnable::run).exceptionally(e -> { final Throwable cause; - if (e.getCause() instanceof ExecutionException) { - cause = e.getCause().getCause(); + if (e.getCause() instanceof final ExecutionException exece) { + cause = exece.getCause(); } else { cause = e.getCause(); } - if (cause instanceof WebApplicationException) { - final WebApplicationException wae = (WebApplicationException) cause; + if (cause instanceof final WebApplicationException wae) { final Response response = wae.getResponse(); String message = ""; - if (wae.getResponse().getEntity() instanceof ErrorPayload) { + if (response.getEntity() instanceof ErrorPayload) { throw wae; // already logged and setup broken so just rethrow } else { try { @@ -212,32 +213,50 @@ private CompletableFuture doExecuteLocalAction(final String family, fi private Response onError(final Throwable re) { log.warn(re.getMessage(), re); - if (re.getCause() instanceof WebApplicationException) { - return ((WebApplicationException) re.getCause()).getResponse(); + if (re instanceof final WebApplicationException webException) { + return webException.getResponse(); + } else if (re.getCause() instanceof final WebApplicationException webException) { + return webException.getResponse(); } - if (re instanceof ComponentException) { - final ComponentException ce = (ComponentException) re; + final String description = "Action execution failed with: " + ofNullable(re.getMessage()) + .orElseGet(() -> re instanceof NullPointerException + ? "unexpected null" + : "no error message"); + if (re instanceof final DiscoverSchemaException eSchema) { + // we send reason to recognize the error on client side + final String subCode = ofNullable(eSchema.getPossibleHandleErrorWith()) + .orElse(HandleErrorWith.EXCEPTION) + .toString(); throw new WebApplicationException(Response - .status(ce.getErrorOrigin() == ComponentException.ErrorOrigin.USER ? 400 - : ce.getErrorOrigin() == ComponentException.ErrorOrigin.BACKEND ? 456 : 520, - "Unexpected callback error") - .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, - "Action execution failed with: " + ofNullable(re.getMessage()) - .orElseGet(() -> re instanceof NullPointerException ? "unexpected null" - : "no error message"))) + .status(400, subCode) + .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, subCode, description)) + .build()); + } else if (re instanceof final ComponentException eComponent) { + throw new WebApplicationException(Response + .status(evaluateStatusCodeForException(eComponent), "Unexpected callback error") + .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, description)) .build()); } throw new WebApplicationException(Response .status(520, "Unexpected callback error") - .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, - "Action execution failed with: " + ofNullable(re.getMessage()) - .orElseGet(() -> re instanceof NullPointerException ? "unexpected null" - : "no error message"))) + .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, description)) .build()); } + private static int evaluateStatusCodeForException(final ComponentException eComponent) { + if (null == eComponent.getErrorOrigin()) { + return 520; + } + + return switch (eComponent.getErrorOrigin()) { + case USER -> 400; + case BACKEND -> 456; + default -> 520; + }; + } + private Stream findVirtualActions(final Predicate typeMatcher, final Predicate componentMatcher, final Locale locale) { return virtualActions diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index 40f1803496e88..113529eb9370b 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -35,6 +36,7 @@ import javax.inject.Inject; import javax.json.bind.Jsonb; +import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.runtime.internationalization.ParameterBundle; import org.talend.sdk.component.runtime.manager.ParameterMeta; import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.ValidationParameterEnricher; @@ -69,54 +71,89 @@ public Stream buildProperties(final List buildProperties(final List meta, final ClassLoader loader, final Locale locale, final DefaultValueInspector.Instance rootInstance, final ParameterMeta parent) { - return meta.stream().flatMap(p -> { - final String path = sanitizePropertyName(p.getPath()); - final String name = sanitizePropertyName(p.getName()); - final String type = p.getType().name(); - final boolean isEnum = p.getType() == ParameterMeta.Type.ENUM; - PropertyValidation validation = propertyValidationService.map(p.getMetadata()); - if (isEnum) { - if (validation == null) { - validation = new PropertyValidation(); - } - validation.setEnumValues(p.getProposals()); - } - final Map sanitizedMetadata = ofNullable(p.getMetadata()) - .map(m -> m - .entrySet() - .stream() - .filter(e -> !e.getKey().startsWith(ValidationParameterEnricher.META_PREFIX)) - .collect(toLinkedMap(e -> e.getKey().replace("tcomp::", ""), Map.Entry::getValue))) - .orElse(null); - final Map metadata; - if (parent != null) { - metadata = sanitizedMetadata; - } else { - metadata = ofNullable(sanitizedMetadata).orElseGet(HashMap::new); - metadata.put("definition::parameter::index", String.valueOf(meta.indexOf(p))); - } - final DefaultValueInspector.Instance instance = defaultValueInspector - .createDemoInstance( - ofNullable(rootInstance).map(DefaultValueInspector.Instance::getValue).orElse(null), p); - final ParameterBundle bundle = p.findBundle(loader, locale); - final ParameterBundle parentBundle = parent == null ? null : parent.findBundle(loader, locale); - return Stream - .concat(Stream - .of(new SimplePropertyDefinition(path, name, - bundle.displayName(parentBundle).orElse(p.getName()), type, toDefault(instance, p), - validation, rewriteMetadataForLocale(metadata, parentBundle, bundle), - bundle.placeholder(parentBundle).orElse(p.getName()), - !isEnum ? null - : p - .getProposals() - .stream() - .collect(toLinkedMap(identity(), - key -> bundle - .enumDisplayName(parentBundle, key) - .orElse(key))))), - buildProperties(p.getNestedParameters(), loader, locale, instance, p)); - }).sorted(Comparator.comparing(SimplePropertyDefinition::getPath)); // important cause it is the way you want to - // see it + return meta.stream() + .flatMap(p -> buildProperty(p, meta, loader, locale, rootInstance, parent)) + // important cause it is the way you want to see it + .sorted(Comparator.comparing(SimplePropertyDefinition::getPath)); + } + + private Stream buildProperty(final ParameterMeta p, + final List siblings, + final ClassLoader loader, + final Locale locale, + final DefaultValueInspector.Instance rootInstance, + final ParameterMeta parent) { + final String path = sanitizePropertyName(p.getPath()); + final String name = sanitizePropertyName(p.getName()); + final String type = p.getType().name(); + + final PropertyValidation validation = buildValidation(p); + final Map metadata = buildMetadata(p, siblings, parent); + + final DefaultValueInspector.Instance instance = defaultValueInspector + .createDemoInstance(ofNullable(rootInstance) + .map(DefaultValueInspector.Instance::getValue) + .orElse(null), p); + final ParameterBundle bundle = p.findBundle(loader, locale); + final ParameterBundle parentBundle = parent == null ? null : parent.findBundle(loader, locale); + final String displayName = bundle.displayName(parentBundle).orElse(p.getName()); + final String placeholder = bundle.placeholder(parentBundle).orElse(p.getName()); + + final LinkedHashMap enumValues = buildEnumDisplayNames(p, bundle, parentBundle); + final SimplePropertyDefinition def = new SimplePropertyDefinition(path, name, displayName, type, + toDefault(instance, p), validation, rewriteMetadataForLocale(metadata, parentBundle, bundle), + placeholder, enumValues); + + return Stream.concat( + Stream.of(def), + buildProperties(p.getNestedParameters(), loader, locale, instance, p)); + } + + private PropertyValidation buildValidation(final ParameterMeta p) { + PropertyValidation validation = propertyValidationService.map(p.getMetadata()); + if (p.getType() != ParameterMeta.Type.ENUM) { + return validation; + } + if (validation == null) { + validation = new PropertyValidation(); + } + validation.setEnumValues(p.getProposals()); + return validation; + } + + private Map buildMetadata( + final ParameterMeta p, + final List siblings, + final ParameterMeta parent) { + final Map sanitized = ofNullable(p.getMetadata()) + .map(m -> m + .entrySet() + .stream() + .filter(e -> !e.getKey().startsWith(ValidationParameterEnricher.META_PREFIX)) + .collect(toLinkedMap(e -> e.getKey().replace("tcomp::", ""), Map.Entry::getValue))) + .orElse(null); + if (parent != null) { + return sanitized; + } + + final Map metadata = ofNullable(sanitized).orElseGet(HashMap::new); + metadata.put("definition::parameter::index", String.valueOf(siblings.indexOf(p))); + // this one to mark the Schema parameter somehow to differentiate it from the branch name + if (p.getJavaType() instanceof Class clazzType && Schema.class.isAssignableFrom(clazzType)) { + metadata.put("definition::parameter::schema", ""); + } + return metadata; + } + + private LinkedHashMap buildEnumDisplayNames(final ParameterMeta p, final ParameterBundle bundle, + final ParameterBundle parentBundle) { + if (p.getType() != ParameterMeta.Type.ENUM) { + return null; + } + + return p.getProposals() + .stream() + .collect(toLinkedMap(identity(), key -> bundle.enumDisplayName(parentBundle, key).orElse(key))); } private Map rewriteMetadataForLocale(final Map metadata, @@ -151,20 +188,21 @@ private Map rewriteLayoutMetadata(final Map meta return metadata; } final Predicate> shouldBeRewritten = k -> keysToRewrite.contains(k.getKey()); - return Stream - .concat(metadata.entrySet().stream().filter(shouldBeRewritten.negate()), - metadata - .entrySet() - .stream() - .filter(shouldBeRewritten) - .map(it -> new AbstractMap.SimpleEntry<>(bundle - .gridLayoutName(parentBundle, - it - .getKey() - .substring("ui::gridlayout::".length(), - it.getKey().length() - "::value".length())) + return Stream.concat( + metadata.entrySet() + .stream() + .filter(shouldBeRewritten.negate()), + metadata.entrySet() + .stream() + .filter(shouldBeRewritten) + .map(it -> new AbstractMap.SimpleEntry<>( + bundle.gridLayoutName(parentBundle, + it.getKey() + .substring("ui::gridlayout::".length(), + it.getKey().length() - "::value".length())) .map(t -> "ui::gridlayout::" + t + "::value") - .orElse(it.getKey()), it.getValue()))) + .orElse(it.getKey()), + it.getValue()))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java index c1b6724a612fb..13b3c32d130b8 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java @@ -64,7 +64,7 @@ class ActionResourceImplTest { void actionIndex() { { // default final ActionList index = base.path("action/index").request(APPLICATION_JSON_TYPE).get(ActionList.class); - assertEquals(13, index.getItems().size()); + assertEquals(14, index.getItems().size()); assertEquals("jdbc", index.getItems().iterator().next().getComponent()); } { // change the family @@ -81,7 +81,7 @@ void actionIndex() { @RepeatedTest(2) void index() { final ActionList index = base.path("action/index").request(APPLICATION_JSON_TYPE).get(ActionList.class); - assertEquals(13, index.getItems().size()); + assertEquals(14, index.getItems().size()); final List items = new ArrayList<>(index.getItems()); items.sort(Comparator.comparing(ActionItem::getName)); @@ -165,6 +165,23 @@ void testBackendException() { assertEquals("Action execution failed with: backend exception", errorPayload.getDescription()); } + @Test + void testDiscoverSchemaException() { + final Response error = base + .path("action/execute") + .queryParam("type", "user") + .queryParam("family", "custom") + .queryParam("action", "discoverSchemaException") + .request(APPLICATION_JSON_TYPE) + .post(Entity.entity(new HashMap(), APPLICATION_JSON_TYPE)); + assertEquals(400, error.getStatus()); + final ErrorPayload errorPayload = error.readEntity(ErrorPayload.class); + assertEquals(ErrorDictionary.ACTION_ERROR, errorPayload.getCode()); + assertEquals("RETRY", errorPayload.getSubCode()); + assertNotNull(errorPayload.getDescription()); + assertTrue(errorPayload.getDescription().startsWith("Action execution failed with:")); + } + @Test void executeWithEnumParam() { final Response error = base @@ -195,14 +212,38 @@ void checkSchemaSerialization() { .request(APPLICATION_JSON_TYPE) .post(Entity.entity(emptyMap(), APPLICATION_JSON_TYPE), String.class); final String expected = - "{\n \"entries\":[\n {\n \"elementSchema\":{\n \"entries\":[\n ],\n" + - " \"metadata\":[\n ],\n \"props\":{\n\n },\n \"type\":\"STRING\"\n" - + - " },\n \"errorCapable\":false," + - "\n \"metadata\":false,\n \"name\":\"array\",\n \"nullable\":false,\n" + - " \"props\":{\n\n },\n \"type\":\"ARRAY\",\n" + - " \"valid\":true\n }\n ],\n \"metadata\":[\n" + - " ],\n \"props\":{\n \"talend.fields.order\":\"array\"\n },\n \"type\":\"RECORD\"\n}"; + """ + { + "entries":[ + { + "elementSchema":{ + "entries":[ + ], + "metadata":[ + ], + "props":{ + + }, + "type":"STRING" + }, + "errorCapable":false, + "metadata":false, + "name":"array", + "nullable":false, + "props":{ + + }, + "type":"ARRAY", + "valid":true + } + ], + "metadata":[ + ], + "props":{ + "talend.fields.order":"array" + }, + "type":"RECORD" + }"""; assertEquals(expected, schema); } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java index 718a3a497a730..f14e83ef73ffa 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java @@ -23,6 +23,7 @@ import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -38,6 +39,7 @@ import org.junit.jupiter.api.Test; import org.talend.sdk.component.api.configuration.Option; import org.talend.sdk.component.api.configuration.ui.layout.GridLayout; +import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.runtime.manager.ParameterMeta; import org.talend.sdk.component.runtime.manager.reflect.ParameterModelService; import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.BaseParameterEnricher; @@ -65,6 +67,14 @@ private static void multipleParams(@Option("aa") final String first, @Option("b" // no-op } + private static void schemaParam(@Option final Schema schema) { + // no-op + } + + private static void nonSchemaParam(@Option final String value) { + // no-op + } + @Test void gridLayoutTranslation() throws NoSuchMethodException { final List params = new ParameterModelService(new PropertyEditorRegistry()) @@ -100,6 +110,30 @@ void parameterIndexMeta() throws NoSuchMethodException { assertEquals("aa/b/a/b.val", props.stream().map(SimplePropertyDefinition::getPath).collect(joining("/"))); } + @Test + void schemaParameterMetadataMarker() throws NoSuchMethodException { + final List schemaParams = new ParameterModelService(new PropertyEditorRegistry()) + .buildParameterMetas(getClass().getDeclaredMethod("schemaParam", Schema.class), null, + new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "test"))); + final List schemaProps = propertiesService + .buildProperties(schemaParams, Thread.currentThread().getContextClassLoader(), Locale.ROOT, null) + .toList(); + assertTrue(schemaProps.stream() + .filter(p -> !p.getPath().contains(".")) + .allMatch(p -> p.getMetadata().containsKey("definition::parameter::schema")), + "Schema-typed root parameter should have definition::parameter::schema marker"); + + final List strParams = new ParameterModelService(new PropertyEditorRegistry()) + .buildParameterMetas(getClass().getDeclaredMethod("nonSchemaParam", String.class), null, + new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "test"))); + final List strProps = propertiesService + .buildProperties(strParams, Thread.currentThread().getContextClassLoader(), Locale.ROOT, null) + .toList(); + assertFalse(strProps.stream() + .anyMatch(p -> p.getMetadata().containsKey("definition::parameter::schema")), + "Non-Schema-typed parameter should NOT have definition::parameter::schema marker"); + } + @Test void booleanDefault() throws NoSuchMethodException { final List props = propertiesService diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomService.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomService.java index 4a656073b25aa..0b2bc09a195ad 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomService.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomService.java @@ -25,6 +25,9 @@ import java.util.stream.Stream; import org.talend.sdk.component.api.exception.ComponentException; +import org.talend.sdk.component.api.exception.ComponentException.ErrorOrigin; +import org.talend.sdk.component.api.exception.DiscoverSchemaException; +import org.talend.sdk.component.api.exception.DiscoverSchemaException.HandleErrorWith; import org.talend.sdk.component.api.service.Action; import org.talend.sdk.component.api.service.Service; import org.talend.sdk.component.api.service.completion.SuggestionValues; @@ -58,16 +61,22 @@ public SuggestionValues get(final LocalConfiguration configuration) throws IOExc @Action("unknownException") public Map generateUnknownException(final LocalConfiguration configuration) { - throw new ComponentException(ComponentException.ErrorOrigin.UNKNOWN, "unknown exception"); + throw new ComponentException(ErrorOrigin.UNKNOWN, "unknown exception"); } @Action("userException") public Map generateUserException(final LocalConfiguration configuration) { - throw new ComponentException(ComponentException.ErrorOrigin.USER, "user exception"); + throw new ComponentException(ErrorOrigin.USER, "user exception"); } @Action("backendException") public Map generateBackendException(final LocalConfiguration configuration) { - throw new ComponentException(ComponentException.ErrorOrigin.BACKEND, "backend exception"); + throw new ComponentException(ErrorOrigin.BACKEND, "backend exception"); + } + + @Action("discoverSchemaException") + public Map generateDiscoverSchemaException(final LocalConfiguration configuration) { + throw new DiscoverSchemaException( + new ComponentException(ErrorOrigin.USER, "schema not found"), HandleErrorWith.RETRY); } } diff --git a/component-studio/component-runtime-di/src/main/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchema.java b/component-studio/component-runtime-di/src/main/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchema.java index fa939a1d9bf79..05659f617a921 100644 --- a/component-studio/component-runtime-di/src/main/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchema.java +++ b/component-studio/component-runtime-di/src/main/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchema.java @@ -17,10 +17,7 @@ import static java.lang.reflect.Modifier.isStatic; import static java.util.Optional.ofNullable; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; import static org.talend.sdk.component.api.exception.DiscoverSchemaException.HandleErrorWith.EXCEPTION; -import static org.talend.sdk.component.api.exception.DiscoverSchemaException.HandleErrorWith.EXECUTE_LIFECYCLE; import static org.talend.sdk.component.api.record.SchemaProperty.IS_KEY; import static org.talend.sdk.component.api.record.SchemaProperty.ORIGIN_TYPE; import static org.talend.sdk.component.api.record.SchemaProperty.PATTERN; @@ -36,10 +33,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.IntStream; @@ -65,12 +60,8 @@ import org.talend.sdk.component.runtime.di.OutputsHandler; import org.talend.sdk.component.runtime.input.Input; import org.talend.sdk.component.runtime.input.Mapper; -import org.talend.sdk.component.runtime.manager.ComponentFamilyMeta; import org.talend.sdk.component.runtime.manager.ComponentManager; import org.talend.sdk.component.runtime.manager.ComponentManager.AllServices; -import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry; -import org.talend.sdk.component.runtime.manager.ParameterMeta; -import org.talend.sdk.component.runtime.manager.ServiceMeta; import org.talend.sdk.component.runtime.manager.chain.ChainedMapper; import org.talend.sdk.component.runtime.manager.xbean.converter.SchemaConverter; import org.talend.sdk.component.runtime.output.InputFactory; @@ -84,23 +75,15 @@ public class TaCoKitGuessSchema { public static final String STRING_ESCAPE = "\""; - public static final String NO_COMPONENT = "No component "; - - public static final String TCOMP_CONFIGURATIONTYPE_TYPE = "tcomp::configurationtype::type"; - - public static final String DATASET = "dataset"; - - public static final String ERROR_THROUGH_ACTION = "Can't guess schema through action."; + public static final String ERROR_THROUGH_ACTION = "Can't guess schema."; public static final String ERROR_NO_AVAILABLE_SCHEMA_FOUND = "There is no available schema found."; - public static final String ERROR_INSTANCE_SCHEMA = "Result is not an instance of Talend Component Kit Schema."; - private static final String NO_COLUMN_FOUND_BY_GUESS_SCHEMA = "No column found by guess schema action"; private ComponentManager componentManager; - private JavaTypesManager javaTypesManager; + private final JavaTypesManager javaTypesManager; private PrintStream out; @@ -122,18 +105,12 @@ public class TaCoKitGuessSchema { private String componentName; - private String action; - private final Integer version; - private static final String SCHEMA_TYPE = "schema"; - - private static final String SCHEMA_EXTENDED_TYPE = "schema_extended"; - private static final String EMPTY = ""; //$NON-NLS-1$ public TaCoKitGuessSchema(final PrintStream out, final Map configuration, final String plugin, - final String family, final String componentName, final String action, final String version) { + final String family, final String componentName, final String version) { this.out = out; this.lineLimit = 50; this.lineCount = -1; @@ -143,11 +120,10 @@ public TaCoKitGuessSchema(final PrintStream out, final Map confi this.plugin = plugin; this.family = family; this.componentName = componentName; - this.action = action; this.columns = new LinkedHashMap<>(); this.keysNoTypeYet = new HashSet<>(); this.javaTypesManager = new JavaTypesManager(); - this.version = Optional.ofNullable(version).map(Integer::parseInt).orElse(null); + this.version = ofNullable(version).map(Integer::parseInt).orElse(null); initClass2JavaTypeMap(); } @@ -159,7 +135,7 @@ public TaCoKitGuessSchema() { private void initClass2JavaTypeMap() { class2JavaTypeMap = new HashMap<>(); - JavaType javaTypes[] = javaTypesManager.getJavaTypes(); + JavaType[] javaTypes = javaTypesManager.getJavaTypes(); for (JavaType javaType : javaTypes) { Class nullableClass = javaType.getNullableClass(); if (nullableClass != null) { @@ -173,15 +149,9 @@ private void initClass2JavaTypeMap() { } private DiscoverSchemaException transformException(final Exception e) { - DiscoverSchemaException discoverSchemaException; - if (e instanceof DiscoverSchemaException) { - discoverSchemaException = (DiscoverSchemaException) e; - } else if (e instanceof ComponentException) { - discoverSchemaException = new DiscoverSchemaException((ComponentException) e); - } else { - discoverSchemaException = new DiscoverSchemaException(e.getMessage(), e.getStackTrace(), EXCEPTION); - } - return discoverSchemaException; + return e instanceof final ComponentException ce + ? new DiscoverSchemaException(ce) + : new DiscoverSchemaException(e.getMessage(), e.getStackTrace(), EXCEPTION); } private DiscoverSchemaException handleException(final Exception e) throws Exception { @@ -194,11 +164,8 @@ private DiscoverSchemaException handleException(final Exception e) throws Except return discoverSchemaException; } - public void guessInputComponentSchema(final Schema schema) throws Exception { + public void guessInputComponentSchema() throws Exception { try { - if (guessSchemaThroughAction(schema)) { - return; - } if (guessInputComponentSchemaThroughResult()) { return; } @@ -208,281 +175,15 @@ public void guessInputComponentSchema(final Schema schema) throws Exception { throw handleException(new Exception(ERROR_NO_AVAILABLE_SCHEMA_FOUND)); } - public void guessComponentSchema(final Schema incomingSchema, final String outgoingBranch, - final boolean isStartOfJob) throws Exception { + /** + * When a processor is the start of a studio job and dev explicitly set the handleError to Lifecycle exec + */ + public void guessComponentSchemaByLifecycle() throws Exception { try { - executeDiscoverSchemaExtendedAction(incomingSchema, outgoingBranch); - } catch (Exception e) { - final DiscoverSchemaException dse = transformException(e); - // When a processor is the start of a studio job and dev explicitly set the handleError to Lifecycle exec - if (isStartOfJob && EXECUTE_LIFECYCLE == dse.getPossibleHandleErrorWith()) { - try { - guessOutputComponentSchemaThroughResult(); - } catch (Exception er) { - throw handleException(e); - } - } else { - throw handleException(e); - } - } - } - - public void guessComponentSchema(final Schema incomingSchema, final String outgoingBranch) throws Exception { - guessComponentSchema(incomingSchema, outgoingBranch, false); - } - - private void executeDiscoverSchemaExtendedAction(final Schema schema, final String branch) throws Exception { - final Collection services = getPluginServices(); - ServiceMeta.ActionMeta actionRef = services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && - a.getType().equals(SCHEMA_EXTENDED_TYPE) && - componentName.equals(a.getAction())) - .findFirst() - .orElse(null); - // did not find action named like componentName, trying to find one matching action... - if (actionRef == null) { - actionRef = services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getType().equals(SCHEMA_EXTENDED_TYPE)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No action " + family + "#" + SCHEMA_EXTENDED_TYPE)); - } - final Object schemaResult = - actionRef.getInvoker().apply(buildActionConfig(actionRef, configuration, schema, branch)); - if (schemaResult instanceof Schema) { - final Schema result = (Schema) schemaResult; - if (result.getEntries().isEmpty()) { - throw new DiscoverSchemaException(ERROR_NO_AVAILABLE_SCHEMA_FOUND, EXCEPTION); - } else { - fromSchema((Schema) schemaResult); - } - } - } - - private Map buildActionConfig(final ServiceMeta.ActionMeta action, - final Map configuration, final Schema schema, final String branch) { - final String schemaPath = action.getParameters() - .get() - .stream() - .filter(p -> Schema.class.isAssignableFrom((Class) p.getJavaType())) - .map(p -> p.getPath()) - .findFirst() - .orElse(""); - final String branchPath = action.getParameters() - .get() - .stream() - .filter(p -> String.class.isAssignableFrom((Class) p.getJavaType())) - .map(ParameterMeta::getPath) - .findFirst() - .orElse(""); - - final Map mapped = new HashMap<>(); - if (!schemaPath.isEmpty()) { - try (final Jsonb jsonb = JsonbBuilder.create()) { - mapped.put(schemaPath, jsonb.toJson(schema)); - } catch (final Exception e) { - throw new IllegalStateException(e); - } + guessOutputComponentSchemaThroughResult(); + } catch (Exception er) { + throw handleException(er); } - if (!branchPath.isEmpty()) { - mapped.put(branchPath, branch); - } - if (configuration == null || configuration.isEmpty()) { - return mapped; - } - final String prefix = action - .getParameters() - .get() - .stream() - .filter(s -> !s.getPath().equals(schemaPath) && !s.getPath().equals(branchPath)) - .map(ParameterMeta::getPath) - .findFirst() - .orElse(null); - if (prefix == null) { - return mapped; - } - mapped.putAll(configuration - .entrySet() - .stream() - .filter(e -> isChildParameter(e.getKey(), prefix) || prefix.equals(e.getKey())) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))); - return mapped; - } - - private Map buildActionConfig(final ServiceMeta.ActionMeta action, - final Map configuration) { - if (configuration == null || configuration.isEmpty()) { - return configuration; // no-mapping - } - - final String prefix = action - .getParameters() - .get() - .stream() - .filter(param -> param.getMetadata().containsKey(TCOMP_CONFIGURATIONTYPE_TYPE) - && DATASET.equals(param.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE))) - .findFirst() - .map(ParameterMeta::getPath) - .orElse(null); - - if (prefix == null) { // no mapping to do - return configuration; - } - - final ParameterMeta dataSet = findDataset(action) - .orElseThrow(() -> new IllegalArgumentException("Dataset not found for " + action.getAction())); - - final String dataSetPath = dataSet.getPath(); - return configuration - .entrySet() - .stream() - .filter(e -> isChildParameter(e.getKey(), dataSetPath)) - .collect(toMap(e -> prefix + e.getKey().substring(dataSetPath.length()), Map.Entry::getValue)); - } - - private boolean isChildParameter(final String path, final String parentPath) { - return path.startsWith(parentPath) && path.substring(parentPath.length()).startsWith("."); - } - - private Optional findDataset(final ServiceMeta.ActionMeta action) { - final ComponentFamilyMeta familyMeta = findFamily(); - final ComponentFamilyMeta.BaseMeta componentMeta = findComponent(familyMeta); - - // dataset name should be the same as DiscoverSchema action name - final Collection metas = toStream(componentMeta.getParameterMetas().get()).collect(toList()); - return ofNullable(metas - .stream() - .filter(p -> DATASET.equals(p.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE)) - && action.getAction().equals(p.getMetadata().get("tcomp::configurationtype::name"))) - .findFirst() - .orElseGet(() -> { - // find and use single dataset - final Iterator iterator = metas - .stream() - .filter(p -> DATASET.equals(p.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE))) - .iterator(); - if (iterator.hasNext()) { - final ParameterMeta value = iterator.next(); - if (!iterator.hasNext()) { - return value; - } - log - .warn("Multiple potential datasets for {}:{}, ignoring parameters", action.getType(), - action.getAction()); - } - return null; - })); - } - - private ComponentFamilyMeta.BaseMeta findComponent(final ComponentFamilyMeta familyMeta) { - return Stream - .concat(familyMeta.getPartitionMappers().entrySet().stream(), - familyMeta.getProcessors().entrySet().stream()) - .filter(e -> e.getKey().equals(componentName)) - .map(Map.Entry::getValue) - .findFirst() - .orElseThrow(() -> new IllegalStateException(NO_COMPONENT + componentName)); - } - - private ComponentFamilyMeta findFamily() { - return componentManager - .findPlugin(plugin) - .orElseThrow(() -> new IllegalArgumentException("No component family " + plugin)) - .get(ContainerComponentRegistry.class) - .getComponents() - .get(family); - } - - private Stream toStream(final Collection parameterMetas) { - return Stream - .concat(parameterMetas.stream(), - parameterMetas - .stream() - .map(ParameterMeta::getNestedParameters) - .filter(Objects::nonNull) - .flatMap(this::toStream)); - } - - private Optional findFirstComponentDataSetName() { - final ComponentFamilyMeta familyMeta = findFamily(); - final ComponentFamilyMeta.BaseMeta componentMeta = findComponent(familyMeta); - return toStream(componentMeta.getParameterMetas().get()) - .filter(p -> DATASET.equals(p.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE))) - .findFirst() - .map(p -> p.getMetadata().get("tcomp::configurationtype::name")); - } - - public boolean guessSchemaThroughAction(final Schema schema) { - final Collection services = getPluginServices(); - - ServiceMeta.ActionMeta actionRef; - if (action == null || action.isEmpty()) { - // Dataset name should match the DiscoverSchema or DiscoverSchemaExtended action name, so try to - // guess it from the component (preferring DiscoverSchemaExtended over DiscoverSchema). - actionRef = findFirstComponentDataSetName() - .flatMap(datasetName -> services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getType().equals(SCHEMA_EXTENDED_TYPE)) - .filter(a -> a.getAction().equals(datasetName)) - .findFirst()) - .orElse(null); - if (actionRef == null) { - // Then try the DiscoverSchema action name. - actionRef = findFirstComponentDataSetName() - .flatMap(datasetName -> services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getType().equals(SCHEMA_TYPE)) - .filter(a -> a.getAction().equals(datasetName)) - .findFirst()) - .orElse(null); - } - } else { - actionRef = services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getAction().equals(action) - && (a.getType().equals(SCHEMA_EXTENDED_TYPE) || a.getType().equals(SCHEMA_TYPE))) - // When both DiscoverSchemaExtended and DiscoverSchema exist for the same action name, - // prefer the extended schema action by ordering it first. - .sorted((action1, action2) -> { - boolean action1IsExtended = action1.getType().equals(SCHEMA_EXTENDED_TYPE); - boolean action2IsExtended = action2.getType().equals(SCHEMA_EXTENDED_TYPE); - return Boolean.compare(!action1IsExtended, !action2IsExtended); - }) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No action " + family + "#(" + SCHEMA_TYPE + " or " + SCHEMA_EXTENDED_TYPE + ")#" - + action)); - } - if (actionRef == null) { - return false; - } - final Map actionConfiguration = - SCHEMA_TYPE.equals(actionRef.getType()) ? buildActionConfig(actionRef, configuration) - : buildActionConfig(actionRef, configuration, schema, "INPUT"); - final Object schemaResult = actionRef.getInvoker().apply(actionConfiguration); - - if (schemaResult instanceof Schema) { - return fromSchema((Schema) schemaResult); - - } else { - log.error(ERROR_INSTANCE_SCHEMA); - return false; - } - } - - private Collection getPluginServices() { - return componentManager - .findPlugin(plugin) - .orElseThrow(() -> new IllegalArgumentException(NO_COMPONENT + plugin)) - .get(ContainerComponentRegistry.class) - .getServices(); } private boolean fromSchema(final Schema schema) { @@ -501,8 +202,7 @@ private boolean fromSchema(final Schema schema) { public Collection getFixedSchema(final String execute) { SchemaConverter sc = new SchemaConverter(); Object o = sc.toObjectImpl(execute); - if (o instanceof Schema) { - final Schema schema = (Schema) o; + if (o instanceof final Schema schema) { final Collection entries = schema.getEntries(); if (entries == null || entries.isEmpty()) { log.info(NO_COLUMN_FOUND_BY_GUESS_SCHEMA); @@ -545,7 +245,7 @@ private Column createColumnFromEntry(final Schema.Entry entry) { column.setComment(entry.getComment()); parseInteger(entry.getProps().get(SIZE)).ifPresent(column::setLength); parseInteger(entry.getProps().get(SCALE)).ifPresent(column::setPrecision); - Optional.ofNullable(entry.getProps().get(IS_KEY)) + ofNullable(entry.getProps().get(IS_KEY)) .ifPresent(value -> column.setKey(Boolean.parseBoolean(value))); if (entryType == Schema.Type.DATETIME || talendType.equals(StudioTypes.DYNAMIC)) { diff --git a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java index c5eb29834c5b0..8fd73d5a7f427 100644 --- a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java +++ b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java @@ -19,19 +19,17 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.talend.sdk.component.api.record.SchemaProperty.IS_KEY; -import static org.talend.sdk.component.api.record.SchemaProperty.PATTERN; import static org.talend.sdk.component.api.record.SchemaProperty.SCALE; import static org.talend.sdk.component.api.record.SchemaProperty.SIZE; -import static org.talend.sdk.component.api.record.SchemaProperty.STUDIO_TYPE; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; import java.io.Serializable; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -43,6 +41,7 @@ import javax.json.bind.JsonbBuilder; import org.apache.beam.sdk.options.Description; +import org.jspecify.annotations.NonNull; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -64,8 +63,6 @@ import org.talend.sdk.component.api.processor.Processor; import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema; -import org.talend.sdk.component.api.record.Schema.Entry; -import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.Service; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.api.service.schema.DiscoverSchemaExtended; @@ -79,15 +76,11 @@ class TaCoKitGuessSchemaTest { private static final String EXPECTED_ERROR_MESSAGE = "Should not be invoked"; - private static RecordBuilderFactory factory; - private final Pattern errorPattern = Pattern.compile("(\\{\"localizedMessage\":\".*?\"possibleHandleErrorWith\":\"\\w+\"})"); private final Pattern schemaPattern = Pattern.compile("(\\[\\{.*\"talendType\".*\\}])"); - private final Pattern logPattern = Pattern.compile("^\\[\\s*(INFO|WARN|ERROR|DEBUG|TRACE)\\s*]"); - private final static java.io.PrintStream stdout = System.out; @BeforeAll @@ -96,7 +89,6 @@ static void forceManagerInit() { if (!manager.find(Stream::of).findAny().isPresent()) { manager.addPlugin(new File("target/test-classes").getAbsolutePath()); } - factory = manager.getRecordBuilderFactoryProvider().apply("default"); } @Description("What are we testing here? " + @@ -110,24 +102,17 @@ void guessSchemaUseVersion(String version) throws Exception { // version is the same, higher than defined in the component or null try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { + final PrintStream out = newPrintStream(byteArrayOutputStream)) { final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema( out, Collections.singletonMap("para1", "bla"), "test-classes", "TaCoKitGuessSchemaTest", "inputDi", - null, version) { - @Override - public boolean guessSchemaThroughAction(final Schema schema) { - // stub to invoke: guessInputComponentSchemaThroughResult - return false; - } - }; - guessSchema.guessInputComponentSchema(null); + guessSchema.guessInputComponentSchema(); guessSchema.close(); assertTrue(byteArrayOutputStream.size() > 0); @@ -135,8 +120,8 @@ public boolean guessSchemaThroughAction(final Schema schema) { final String content = byteArrayOutputStream.toString(); assertTrue(content.contains("\"length\":10")); assertTrue(content.contains("\"precision\":2")); - assertTrue(!content.contains("\"length\":0")); - assertTrue(!content.contains("\"precision\":0")); + assertFalse(content.contains("\"length\":0")); + assertFalse(content.contains("\"precision\":0")); } } @@ -150,116 +135,35 @@ void guessSchemaUseVersionNOK(final String version) throws Exception { // version is lower than defined in the component try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { + final PrintStream out = newPrintStream(byteArrayOutputStream)) { final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema( out, Collections.singletonMap("para1", "bla"), "test-classes", "TaCoKitGuessSchemaTest", "inputDi", - null, version) { - @Override - public boolean guessSchemaThroughAction(final Schema schema) { - // stub to invoke: guessInputComponentSchemaThroughResult - return false; - } - }; final DiscoverSchemaException exception = - Assertions.assertThrows(DiscoverSchemaException.class, - () -> guessSchema.guessInputComponentSchema(null)); + Assertions.assertThrows(DiscoverSchemaException.class, guessSchema::guessInputComponentSchema); assertEquals(EXPECTED_ERROR_MESSAGE, exception.getMessage()); assertEquals(HandleErrorWith.EXCEPTION, exception.getPossibleHandleErrorWith()); } } - final Entry f1 = factory.newEntryBuilder() - .withName("f1") - .withType(Schema.Type.STRING) - .build(); - - final Entry f2 = factory.newEntryBuilder() - .withName("f2") - .withType(Schema.Type.LONG) - .withDefaultValue(11l) - .build(); - - final Entry f3 = factory.newEntryBuilder() - .withName("f3") - .withType(Schema.Type.BOOLEAN) - .build(); - - @Test - void guessProcessorSchema() throws Exception { - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - final Entry f1 = factory.newEntryBuilder() - .withName("f1") - .withType(Schema.Type.STRING) - .withProp(SchemaProperty.ORIGIN_TYPE, "VARCHAR") - .build(); - final Entry f2 = factory.newEntryBuilder() - .withName("f2") - .withType(Schema.Type.LONG) - .withDefaultValue(11l) - .withProp(SchemaProperty.ORIGIN_TYPE, "LONGINT") - .build(); - final Entry f3 = factory.newEntryBuilder() - .withName("f3") - .withType(Schema.Type.BOOLEAN) - .withProp(SchemaProperty.ORIGIN_TYPE, "BOOLEAN") - .build(); - final Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD) - .withProp("aprop", "a property!") - .withEntry(f1) - .withEntry(f2) - .withEntry(f3) - .build(); - Map config = new HashMap<>(); - config.put("configuration.param1", "parameter one"); - config.put("configuration.param2", "parameter two"); - final TaCoKitGuessSchema guessSchema = - new TaCoKitGuessSchema(out, config, "test-classes", "TaCoKitGuessSchemaTest", "outputDi", null, - "1"); - redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", false); - guessSchema.close(); - restoreStdout(); - final String flattened = flatten(byteArrayOutputStream); - final String expected = - "[{\"label\":\"f1\",\"nullable\":false,\"originalDbColumnName\":\"f1\",\"sourceType\":\"VARCHAR\",\"talendType\":\"id_String\"},{\"default\":\"11\",\"defaut\":\"11\",\"label\":\"f2\",\"nullable\":false,\"originalDbColumnName\":\"f2\",\"sourceType\":\"LONGINT\",\"talendType\":\"id_Long\"},{\"label\":\"f3\",\"nullable\":false,\"originalDbColumnName\":\"f3\",\"sourceType\":\"BOOLEAN\",\"talendType\":\"id_Boolean\"},{\"comment\":\"branch name\",\"label\":\"out\",\"nullable\":false,\"originalDbColumnName\":\"out\",\"talendType\":\"id_String\"}]"; - final Matcher schemaMatcher = schemaPattern.matcher(flattened); - assertFalse(errorPattern.matcher(flattened).find()); - assertTrue(schemaMatcher.find()); - assertEquals(expected, schemaMatcher.group()); - } - } - - @Test - void guessProcessorSchemaRecordBuilderFactoryImpl() throws Exception { - guessProcessorSchemaWithRecordBuilderFactory(null); - } - - @Test - void guessProcessorSchemaAvroRecordBuilderFactory() throws Exception { - guessProcessorSchemaWithRecordBuilderFactory(factory); - } - @Test void guessProcessorSchemaInStartOfJob() throws Exception { try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - final Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD).build(); - Map config = new HashMap<>(); + final PrintStream out = newPrintStream(byteArrayOutputStream)) { + final Map config = new HashMap<>(); config.put("configuration.shouldActionFail", "true"); config.put("configuration.failWith", "EXECUTE_LIFECYCLE"); final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(out, config, "test-classes", - "TaCoKitGuessSchemaTest", "outputDi", null, "1"); - // guess schema action will fail and as start of job is true, it should use processor lifecycle + "TaCoKitGuessSchemaTest", "outputDi", "1"); + // guesses schema directly via component lifecycle (no action invocation) redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", true); + guessSchema.guessComponentSchemaByLifecycle(); guessSchema.close(); restoreStdout(); final String expected = @@ -272,221 +176,6 @@ void guessProcessorSchemaInStartOfJob() throws Exception { } } - @Test - void guessProcessorSchemaInStartWithMockExecution() throws Exception { - final Schema sin = new RecordBuilderFactoryImpl("test-classes").newSchemaBuilder(Schema.Type.RECORD).build(); - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - Map config = new HashMap<>(); - config.put("configuration.shouldActionFail", "true"); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(out, config, "test-classes", - "TaCoKitGuessSchemaTest", "outputDi", null, "1"); - try { - redirectStdout(out); - guessSchema.guessComponentSchema(sin, "out", true); - } catch (Exception e) { - guessSchema.close(); - } - restoreStdout(); - // same transformations as in Studio - final String flattened = flatten(byteArrayOutputStream); - final Matcher errorMatcher = errorPattern.matcher(flattened); - assertFalse(schemaPattern.matcher(flattened).find()); - assertTrue(errorMatcher.find()); - final DiscoverSchemaException de = jsonToException(errorMatcher.group()); - assertNotNull(de); - assertEquals("Cannot execute action.", de.getMessage()); - assertEquals(HandleErrorWith.EXECUTE_MOCK_JOB, de.getPossibleHandleErrorWith()); - } - } - - private void guessProcessorSchemaWithRecordBuilderFactory(RecordBuilderFactory facto) throws Exception { - final RecordBuilderFactory factory = facto == null ? new RecordBuilderFactoryImpl("test") : facto; - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD) - .withProp("aprop", "a property!") - .withEntry(f1) - .withEntry(f2) - .withEntry(f3) - .withEntry(factory.newEntryBuilder() - .withName("id") - .withType(Schema.Type.INT) - .withNullable(false) - .withRawName("id") - .withComment("hjk;ljkkj") - .withProp(STUDIO_TYPE, "id_Integer") - .withProp(IS_KEY, "true") - .withProp(SCALE, "0") - .withProp(SIZE, "10") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("name") - .withType(Schema.Type.STRING) - .withNullable(true) - .withRawName("name") - .withComment("hljkjhlk") - .withDefaultValue("toto") - .withProp(STUDIO_TYPE, "id_String") - .withProp(SCALE, "0") - .withProp(SIZE, "20") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("flag") - .withType(Schema.Type.STRING) - .withNullable(true) - .withRawName("flag") - .withProp(STUDIO_TYPE, "id_Character") - .withProp(SCALE, "0") - .withProp(SIZE, "4") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("female") - .withType(Schema.Type.BOOLEAN) - .withNullable(true) - .withRawName("female") - .withProp(STUDIO_TYPE, "id_Boolean") - .withProp(SCALE, "0") - .withProp(SIZE, "1") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("num1") - .withType(Schema.Type.BYTES) - .withNullable(true) - .withRawName("num1") - .withComment("hhhh") - .withProp(STUDIO_TYPE, "id_Byte") - .withProp(SCALE, "0") - .withProp(SIZE, "3") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("num2") - .withType(Schema.Type.INT) - .withNullable(true) - .withRawName("num2") - .withProp(STUDIO_TYPE, "id_Short") - .withProp(SCALE, "0") - .withProp(SIZE, "5") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("age") - .withType(Schema.Type.LONG) - .withNullable(true) - .withRawName("age") - .withProp(STUDIO_TYPE, "id_Long") - .withProp(SCALE, "0") - .withProp(SIZE, "19") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("bonus") - .withType(Schema.Type.FLOAT) - .withNullable(true) - .withRawName("bonus") - .withProp(STUDIO_TYPE, "id_Float") - .withProp(SCALE, "2") - .withProp(SIZE, "12") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("salary") - .withType(Schema.Type.DOUBLE) - .withNullable(true) - .withRawName("salary") - .withProp(STUDIO_TYPE, "id_Double") - .withProp(SCALE, "2") - .withProp(SIZE, "22") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("play") - .withType(Schema.Type.STRING) - .withNullable(true) - .withRawName("play") - .withProp(STUDIO_TYPE, "id_String") - .withProp(SCALE, "2") - .withProp(SIZE, "10") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("startdate") - .withType(Schema.Type.DATETIME) - .withNullable(true) - .withRawName("startdate") - .withProp(STUDIO_TYPE, "id_Date") - .withProp(PATTERN, "yyyy-MM-dd") - .build()) - .build(); - - Map config = new HashMap<>(); - config.put("configuration.param1", "parameter one"); - config.put("configuration.param2", "parameter two"); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema( - out, config, "test-classes", "TaCoKitGuessSchemaTest", - "outputDi", null, "1"); - redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", false); - guessSchema.close(); - restoreStdout(); - - final String flattened = flatten(byteArrayOutputStream); - final String expected = - "[{\"label\":\"f1\",\"nullable\":false,\"originalDbColumnName\":\"f1\",\"talendType\":\"id_String\"},{\"default\":\"11\",\"defaut\":\"11\",\"label\":\"f2\",\"nullable\":false,\"originalDbColumnName\":\"f2\",\"talendType\":\"id_Long\"},{\"label\":\"f3\",\"nullable\":false,\"originalDbColumnName\":\"f3\",\"talendType\":\"id_Boolean\"},{\"comment\":\"hjk;ljkkj\",\"key\":true,\"label\":\"id\",\"length\":10,\"nullable\":false,\"originalDbColumnName\":\"id\",\"precision\":0,\"talendType\":\"id_Integer\"},{\"comment\":\"hljkjhlk\",\"default\":\"toto\",\"defaut\":\"toto\",\"label\":\"name\",\"length\":20,\"nullable\":true,\"originalDbColumnName\":\"name\",\"precision\":0,\"talendType\":\"id_String\"},{\"label\":\"flag\",\"length\":4,\"nullable\":true,\"originalDbColumnName\":\"flag\",\"precision\":0,\"talendType\":\"id_Character\"},{\"label\":\"female\",\"length\":1,\"nullable\":true,\"originalDbColumnName\":\"female\",\"precision\":0,\"talendType\":\"id_Boolean\"},{\"comment\":\"hhhh\",\"label\":\"num1\",\"length\":3,\"nullable\":true,\"originalDbColumnName\":\"num1\",\"precision\":0,\"talendType\":\"id_Byte\"},{\"label\":\"num2\",\"length\":5,\"nullable\":true,\"originalDbColumnName\":\"num2\",\"precision\":0,\"talendType\":\"id_Short\"},{\"label\":\"age\",\"length\":19,\"nullable\":true,\"originalDbColumnName\":\"age\",\"precision\":0,\"talendType\":\"id_Long\"},{\"label\":\"bonus\",\"length\":12,\"nullable\":true,\"originalDbColumnName\":\"bonus\",\"precision\":2,\"talendType\":\"id_Float\"},{\"label\":\"salary\",\"length\":22,\"nullable\":true,\"originalDbColumnName\":\"salary\",\"precision\":2,\"talendType\":\"id_Double\"},{\"label\":\"play\",\"length\":10,\"nullable\":true,\"originalDbColumnName\":\"play\",\"precision\":2,\"talendType\":\"id_String\"},{\"label\":\"startdate\",\"nullable\":true,\"originalDbColumnName\":\"startdate\",\"pattern\":\"\\\"yyyy-MM-dd\\\"\",\"talendType\":\"id_Date\"},{\"comment\":\"branch name\",\"label\":\"out\",\"nullable\":false,\"originalDbColumnName\":\"out\",\"talendType\":\"id_String\"}]"; - final Matcher schemaMatcher = schemaPattern.matcher(flattened); - assertFalse(errorPattern.matcher(flattened).find()); - assertTrue(schemaMatcher.find()); - assertEquals(expected, schemaMatcher.group()); - } - } - - @Test - void testFromSchema() throws Exception { - final RecordBuilderFactory factory = new RecordBuilderFactoryImpl("test-classes"); - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD) - .withEntry(factory.newEntryBuilder() - .withName("name") - .withType(Schema.Type.STRING) - .withProp(STUDIO_TYPE, StudioTypes.STRING) - .build()) - .withEntry(factory.newEntryBuilder() - .withName("bit") - .withType(Schema.Type.BYTES) - .withProp(STUDIO_TYPE, StudioTypes.BYTE) - .build()) - .withEntry(factory.newEntryBuilder() - .withName("dynamic") - .withType(Schema.Type.RECORD) - .withProp(STUDIO_TYPE, StudioTypes.DYNAMIC) - .withProp(PATTERN, "dd/MM/YYYY") - .withElementSchema(factory.newSchemaBuilder(Schema.Type.RECORD).build()) - .build()) - .withEntry(factory.newEntryBuilder() - .withName("document") - .withType(Schema.Type.RECORD) - .withProp(STUDIO_TYPE, StudioTypes.DOCUMENT) - .withElementSchema(factory.newSchemaBuilder(Schema.Type.RECORD).build()) - .build()) - .build(); - - Map config = new HashMap<>(); - config.put("configuration.skipAssertions", "true"); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema( - out, config, "test-classes", "TaCoKitGuessSchemaTest", - "outputDi", null, "1"); - redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", false); - guessSchema.close(); - restoreStdout(); - - final String flattened = flatten(byteArrayOutputStream); - final String expected = - "[{\"label\":\"name\",\"nullable\":false,\"originalDbColumnName\":\"name\",\"talendType\":\"id_String\"},{\"label\":\"bit\",\"nullable\":false,\"originalDbColumnName\":\"bit\",\"talendType\":\"id_Byte\"},{\"label\":\"dynamic\",\"nullable\":true,\"originalDbColumnName\":\"dynamic\",\"pattern\":\"\\\"dd/MM/YYYY\\\"\",\"talendType\":\"id_Dynamic\"},{\"label\":\"document\",\"nullable\":true,\"originalDbColumnName\":\"document\",\"talendType\":\"id_Document\"},{\"comment\":\"branch name\",\"label\":\"out\",\"nullable\":false,\"originalDbColumnName\":\"out\",\"talendType\":\"id_String\"}]"; - assertTrue(byteArrayOutputStream.size() > 0); - final Matcher schemaMatcher = schemaPattern.matcher(flattened); - assertFalse(errorPattern.matcher(flattened).find()); - assertTrue(schemaMatcher.find()); - assertEquals(expected, schemaMatcher.group()); - } - } - @Test void serializeDiscoverSchemaException() throws Exception { final DiscoverSchemaException de = new DiscoverSchemaException( @@ -509,6 +198,7 @@ void deserializeDiscoverSchemaException() throws Exception { final String serialized = "{\"localizedMessage\":\"Unknown error. Retry!\",\"message\":\"Unknown error. Retry!\",\"stackTrace\":[],\"suppressed\":[],\"possibleHandleErrorWith\":\"RETRY\"}"; DiscoverSchemaException e = jsonToException(flattened); + assertNotNull(e.getPossibleHandleErrorWith()); assertNotEquals("EXECUTE_MOCK_JOB", e.getPossibleHandleErrorWith().name()); assertEquals("EXCEPTION", e.getPossibleHandleErrorWith().name()); assertEquals("Not allowed to execute the HTTP call to retrieve the schema.", e.getMessage()); @@ -545,7 +235,7 @@ void restoreStdout() { } private String flatten(ByteArrayOutputStream out) { - return out.toString().replaceAll("\n", ""); + return out.toString().replace("\n", ""); } @Data @@ -562,9 +252,8 @@ public static class TestDataStore implements Serializable { @Emitter(name = "inputDi", family = "TaCoKitGuessSchemaTest") public static class InputComponentDi implements Serializable { - private RecordBuilderFactory factory = new RecordBuilderFactoryImpl("test-classes"); + private final RecordBuilderFactory factory = new RecordBuilderFactoryImpl("test-classes"); - // TODO : in future, will always use action result instead of mock job result, so will remove this @Producer public Record next() { final Schema.Entry entry1 = factory.newEntryBuilder() @@ -646,9 +335,6 @@ public static class ProcessorConfiguration implements Serializable { @Option HandleErrorWith failWith = HandleErrorWith.EXECUTE_MOCK_JOB; - - @Option - Boolean skipAssertions = false; } @Slf4j @@ -676,31 +362,13 @@ public Schema discoverProcessorSchema(final Schema incomingSchema, log.error("[discoverProcessorSchema] Action will fail!"); throw new DiscoverSchemaException("Cannot execute action.", conf.failWith); } - if (!conf.skipAssertions) { - assertEquals("out", branch); - assertEquals("parameter one", conf.param1); - assertEquals("parameter two", conf.param2); - assertNull(conf.param3); - assertNotNull(incomingSchema); - assertEquals(Schema.Type.RECORD, incomingSchema.getType()); - assertEquals("a property!", incomingSchema.getProp("aprop")); - assertNotNull(incomingSchema.getEntry("f1")); - assertEquals(Schema.Type.STRING, incomingSchema.getEntry("f1").getType()); - assertNotNull(incomingSchema.getEntry("f2")); - assertEquals(Schema.Type.LONG, incomingSchema.getEntry("f2").getType()); - assertEquals(11L, (Long) incomingSchema.getEntry("f2").getDefaultValue()); - assertNotNull(incomingSchema.getEntry("f3")); - assertEquals(Schema.Type.BOOLEAN, incomingSchema.getEntry("f3").getType()); - } - return factory.newSchemaBuilder(incomingSchema) - .withEntry(factory.newEntryBuilder() - .withName(branch) - .withType(Schema.Type.STRING) - .withComment("branch name") - .withProp("branch", branch) - .build()) - .build(); + + throw new IllegalStateException(EXPECTED_ERROR_MESSAGE); } } + @NonNull + private static PrintStream newPrintStream(final ByteArrayOutputStream byteArrayOutputStream) { + return new PrintStream(byteArrayOutputStream, false, StandardCharsets.UTF_8); + } } \ No newline at end of file diff --git a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java index f2722066b082f..6c97b2da67f7c 100644 --- a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java +++ b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java @@ -31,7 +31,7 @@ class TypeConversionTest { @Test void jsonValueToTalendType() { JavaTypesManager javaTypesManager = new JavaTypesManager(); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(null, null, null, null, null, null, null); + final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(null, null, null, null, null, null); assertEquals(javaTypesManager.LONG.getId(), guessSchema.getTalendType(jsonProvider.createValue(1))); assertEquals(javaTypesManager.LONG.getId(), guessSchema.getTalendType(jsonProvider.createValue(1L))); assertEquals(javaTypesManager.DOUBLE.getId(), guessSchema.getTalendType(jsonProvider.createValue(1.1d)));