diff --git a/build.gradle b/build.gradle
index 3dd9a17bc..f46d02c80 100644
--- a/build.gradle
+++ b/build.gradle
@@ -86,7 +86,8 @@ def testSchemas = [
[dir:'build/schemas/sTRANXML',path:'tranxml',name:'sTRANXML'],
[dir:'src/test/resources/xbean/xmlobject',path:'CR192525.xsd defaults.xsd easypo.xsd easypo.xsdconfig enumtest.xsd enumtest.xsdconfig inttest.xsd listandunion.xsd nameworld.xsd numerals.xsd redefine2.xsd SelectAttribute.xsd SelectChildren.xsd selectChldAttTest.xsd soap-envelope.xsd soapfaults.xsd substgroup.xsd substgroup2.xsd Test.xsd Test36510.xsd Test40907.xsd version3.xsd', name:'sXMLOBJECT'],
[dir:'src/test/resources/xbean/misc/jira',path:'xmlbeans_33b.xsd xmlbeans_46.xsd xmlbeans_48.xsd xmlbeans_64.xsd xmlbeans_68.xsd xmlbeans_71.xsd xmlbeans_98.xsd xmlbeans_99.xsd xmlbeans_105.xsd xmlbeans_175.xsd xmlbeans_177.xsd xmlbeans_177a.xsd xmlbeans_208.xsd xmlbeans_228.xsd xmlbeans_307_maxallowedenum.xsd xmlbeans_307_morethanallowedenum.xsd', name:'sJIRA' ],
- [dir:'src/test/resources/xbean',path:'xmltokensource',name:'sTOKENSOURCE']
+ [dir:'src/test/resources/xbean',path:'xmltokensource',name:'sTOKENSOURCE'],
+ [dir:'src/test/resources/xbean',path:'labkey',name:'sLABKEY']
] + xsdDirs(file('src/test/resources/xbean/extensions')).withIndex().collect { element, index ->
// the extensions need to be generated separately, because the *.xsdconfigs have overlapping definitions
[dir:'src/test/resources/xbean/extensions',path:element,name:"sEXT${index}"]
diff --git a/build.xml b/build.xml
index 94ebd12f4..9b1682bbb 100644
--- a/build.xml
+++ b/build.xml
@@ -488,6 +488,7 @@
+
diff --git a/src/test/java/org/apache/xmlbeans/impl/schema/XmlBeans661Test.java b/src/test/java/org/apache/xmlbeans/impl/schema/XmlBeans661Test.java
new file mode 100644
index 000000000..a78cb9d4a
--- /dev/null
+++ b/src/test/java/org/apache/xmlbeans/impl/schema/XmlBeans661Test.java
@@ -0,0 +1,56 @@
+/* Copyright 2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xmlbeans.impl.schema;
+
+import org.junit.jupiter.api.Test;
+import org.labkey.data.xml.queryCustomView.FilterType;
+import org.labkey.etl.xml.EtlDocument;
+import org.labkey.etl.xml.EtlType;
+import org.labkey.etl.xml.SourceObjectType;
+import org.labkey.etl.xml.TransformType;
+
+import java.net.URL;
+import java.util.Arrays;
+
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+// test for https://issues.apache.org/jira/browse/XMLBEANS-661
+// lots of classes and related xsds were copied into the test src
+// to avoid the issue of having to regenerate them in the build
+public class XmlBeans661Test {
+
+ @Test
+ public void testXmlBeans661() throws Exception {
+ URL dataUrl = XmlBeans661Test.class.getClassLoader()
+ .getResource("xbean/labkey/SourceToTarget2WithFilter.xml");
+ assertNotNull(dataUrl, "Test data file not found");
+ EtlDocument document = EtlDocument.Factory.parse(dataUrl.openStream());
+ EtlType etlXml = document.getEtl();
+ for (TransformType transformXml : etlXml.getTransforms().getTransformArray()) {
+ SourceObjectType source = transformXml.getSource();
+ assertNotNull(source, "Source not found");
+ assertTrue(source.isSetSourceFilters(), "Source filters not set");
+ // This is where the failure happened with the following error:
+ // Exception in thread "main" java.lang.ArrayStoreException: arraycopy: element type mismatch: can not cast one of the elements of java.lang.Object[] to the type of the destination array, org.labkey.data.xml.queryCustomView.FilterType
+ FilterType[] filterTypes = source.getSourceFilters().getSourceFilterArray();
+ assertNotNull(filterTypes, "Source filters should not be null");
+ assertNotEquals(0, filterTypes.length, "Source filters should not be empty");
+ Arrays.stream(filterTypes).map(Object::toString).forEach(System.out::println);
+ }
+ }
+}
diff --git a/src/test/resources/xbean/labkey/SourceToTarget2WithFilter.xml b/src/test/resources/xbean/labkey/SourceToTarget2WithFilter.xml
new file mode 100644
index 000000000..82f207254
--- /dev/null
+++ b/src/test/resources/xbean/labkey/SourceToTarget2WithFilter.xml
@@ -0,0 +1,50 @@
+
+
+
+ Source to target with sourceFilter
+ append rows from source to target with a sourceFilter
+
+
+ Copy to target
+
+
+
+
+
+
+
+
+
+
+
+
+ Copy to target2
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/test/resources/xbean/labkey/schemas/etl.xsd b/src/test/resources/xbean/labkey/schemas/etl.xsd
new file mode 100644
index 000000000..976af7333
--- /dev/null
+++ b/src/test/resources/xbean/labkey/schemas/etl.xsd
@@ -0,0 +1,578 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ETL can be run on its own, rather than a subcomponent that can only be queued from another ETL.
+
+
+
+
+ ETL can run at a site level scope, not just in a container.
+
+
+
+
+ By default, if a job for the etl is already pending (waiting), we block adding another instance of the etl to the job queue. Set this flag to override and allow multiple instances in the queue.
+
+
+
+
+ Optional. Experimental, Postgres datasources only. Causes a multi-step ETL to wrap an entire run of the ETL in a single transaction
+ on the source datasource containing the schema specified. This transaction will be isolation level REPEATABLE READ, guaranteeing consistent
+ state of source data throughout the transaction. This prevents phantom reads of source data which has been inserted/updated since
+ the ETL started. Only meaningful if the source schemas for every step in the ETL are all from the same datasource. Individual steps
+ will not have their own transaction boundaries.
+ The intended use case is for a source datasource external to LabKey Server.
+ This setting is independent of the optional transactDestinationSchema setting. If both are specified, and the two schemas are from
+ the same datasource, the ETL will be wrapped in a single REPEATABLE READ transaction. This scenario is only supported if
+ tha single datasource is Postgres, and is not recommended.
+
+
+
+
+
+ Optional. Very experimental, use with caution. Causes a multi-step ETL to wrap an entire run of the ETL in a single transaction
+ on the destination datasource containing the schema specified. This transaction will only be committed upon successful completion
+ of the ETL run; any error will cause a full rollback of every step up to that point. Only meaningful if the destination schemas for every
+ step in the ETL are all from the same datasource. Individual steps will not have their own transaction boundaries, nor will setting
+ a batchSize have any effect. Using this setting will very likely cause lock contention problems on the destination queries,
+ especially on SQL Server.
+ This setting is independent of the optional transactSourceSchema setting. If both are specified, and the two schemas are from
+ the same datasource, the ETL run will be wrapped in a single REPEATABLE READ transaction. This scenario is only supported if
+ tha single datasource is Postgres, and is not recommended.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Container path for doing cross-container ETLs. If not entered, defaults to the container where the ETL is running.
+
+
+
+
+
+
+ Override the column name set on a ModifiedSinceFilter incremental filter.
+
+
+
+
+ Override the column name set on a RunFilter incremental filter.
+
+
+
+
+
+
+ Wrap selecting from source query in a transaction. Only used for simple query transform types.
+
+
+
+
+ Specify the type of container filter to use on the source query.
+
+
+
+
+
+
+ Targets will be of type 'query' (the default), or 'file'. For query, the schemaName and queryName attributes are required. For file, the fileBaseName and fileExtension attributes are required.
+
+
+
+
+
+
+
+
+
+
+ Bulk loads minimize the logging for auditing purposes and other overhead.
+
+
+
+
+
+ The base name to use for an output target file. Special substitutions:
+ '${TransformRunId}' will be substituted with the transformRunId.
+ '${Timestamp}' will be substituted with the timestamp at file creation.
+
+
+
+
+ Optional (required for pipeline usage), the extension for the output target file. A leading dot is optional; there will always be a dot separator between the file basename and extension
+
+
+
+
+
+ Character to qualify text when a field contains the column or row delimiter
+
+
+
+
+
+
+
+ Wrap writing to target query in a transaction. Not used when target is a file.
+
+
+
+
+
+ Formerly called "transactionSize". For query targets, incrementally commit target transaction every n rows instead of in a single wrapping transaction.
+ For file targets, instead of writing a single file, create a new file every n rows. Each will be the fileBaseName + "-1", "-2", etc, + fileExtension
+
+
+
+
+ If specified, rather than count individual rows for the batchSize, use this as a sentinel field, and only increment the count of a current batch when the value in this field changes.
+ If the name of the relevant column is being mapped via columnTransforms, use the target name, not the source name.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ The target will be a file of basename and extension specified by fileBaseName and fileExtension.
+ By default the field and row separators will be for a tsv file, but this can be overridden with the optional columnDelimiter and rowDelimiter attributes.
+ For ETLs that use the target file as input to a pipeline command tasks, it is possible to skip running the command task for a 0 row file. Use the "etlOutputHadRows"
+ parameter in the pipeline configuration. See nlpContext.xml for an example of this.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Wrap the call of the procedure in a transaction.
+
+
+
+
+
+
+
+
+
+
+
+ When present, this procedure will be used to check if there is work for the job to do. If the output value
+ of this parameter is equal to the noWorkValue, there is no work for the job to do.
+ This can either be an actual value ("4"), or there is a substitution syntax to indicate comparison should be against the
+ input value of a certain parameter. E.g., a parameter name="batchId" noWorkValue="${batchId}" will indicate there
+ is no work for the job if the output batchId is the same as the one saved from the previous run.
+
+
+
+
+
+
+
+ These match two of the scopes defined in the Variable Map persisted in the TransformConfiguration.TransformState field.
+ (We don't support the notion of a 'parent' scope here.) Globally scoped parameters allow sharing/passing of context across multiple stored procedure steps.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Use 'StoredProcedure' when handling the transform task wtih a stored procedure.
+ Use 'ExternalPipelineTask' (in conjunction with externalTaskId) when handling the transform task with an XML-defined pipeline task.
+ Use 'TaskrefTransformStep' (in conjunction with a taskRef element) to queue another ETL to run.
+
+
+
+
+
+
+ The command task (wrapped as a pipeline task) that handles the transform.
+ E.g., 'org.labkey.api.pipeline.cmd.CommandTask:myEngineCommand'
+
+
+
+
+
+
+ If an externalTaskId command task is defined, settings from this pipeline
+ (to date, workflow process key) will be applied to the pipeline wrapping the externalTask
+ E.g., 'org.labkey.api.pipeline.file.FileAnalysisTaskPipeline:myPipelineName'
+
+
+
+
+
+
+ Persist the job state to the database after this step (in addition to at the end of a successfully complete job).
+ Use with extreme caution; if a later step causes an error in the job, this will still be the assumed state for the next run.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Run the ETL at the specified interval, for example, 2h to run every two hours.
+ Valid intervals are:
+ y: Years
+ m: Months, if day or hour is specified
+ d: Days
+ h: Hours
+ m: Minutes, if no day or hour is specified
+ s: Seconds
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ The column in the deletedRowsSource which holds the key values to delete in the target. If not specified, the PK values of the delete query is used.
+
+
+
+
+
+
+ The column in the target query corresponding to the deleted key values. If not specified, we assume the PK of the target query.<br/>
+ If a non-PK column is specified, the delete values will be used as lookups to map back to the corresponding PK values in the target.
+
+
+
+
+
+
+
+
+
+
+
+ Defines a query with a list of rows to delete from the target in this incremental run.<br/>
+ Timestamp or run filter values against the delete rows source are tracked independently from the values in the<br/>
+ original source query. I.e., there could be no new records in the source query, but new records in the delete rows source<br/>
+ will still be found and deleted from the target.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Container path for doing cross-container run filter strategy. If not specified, defaults to the container where the ETL is running.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Intended for use when the ETL's source query is a parameterized SQL query.
+ Parameter values placed here are passed into the SQL query.
+ For details see
+ https://www.labkey.org/Documentation/wiki-page.view?name=paramsql
+ and
+ https://www.labkey.org/Documentation/wiki-page.view?name=etlSamples#param
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Enumerated list of java.sql.types. Mirrors the enum defined in java JdbcType.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Intended for use when an ETL invokes file analysis pipeline command tasks.
+ Will be passed as parameter overrides into the protocol.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Defines a set of alternate match keys to use for merge ETLs instead of the primary key of the target.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Defines a java class to use to transform a column value.
+ This works for both query and file targets.
+ The implementation of the transform class declares if "source" and/or "target"
+ column names are required. Default implementation requires only source, and the
+ same name is used for the target column.
+ If no transformClass is specified, a simple in-flight column name mapping is
+ performed between the source and target. In this case both source and target are required.
+
+
+
+
+
+
+
+
+
+
+ A list of fields with constant values that will be added to the source fields.
+ If there is a column with that name in the source query, the value provided here will
+ be an override.
+ These can be defined at the global etl level, applying to all steps, or at the individual step
+ (destination) level. If the same column is specified both globally and in an individual step,
+ that step will receive the value defined at the step level.
+ The form of these are the same as for source query parameters. See
+ https://www.labkey.org/Documentation/wiki-page.view?name=etlSamples#param
+
+
+
+
+
+
+
diff --git a/src/test/resources/xbean/labkey/schemasCustomView/queryCustomView.xsd b/src/test/resources/xbean/labkey/schemasCustomView/queryCustomView.xsd
new file mode 100644
index 000000000..5b9c129bf
--- /dev/null
+++ b/src/test/resources/xbean/labkey/schemasCustomView/queryCustomView.xsd
@@ -0,0 +1,234 @@
+
+
+
+
+ File-based Query Custom View Definition
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ The current container.
+
+
+ Current folder with permissions applied to user.
+
+
+ Current folder and first children that are not workbooks.
+
+
+ The current container and any folders it contains.
+
+
+ Current folder and siblings.
+
+
+ Current folder and/or parent if the current folder is a workbook, plus all workbooks in this series.
+
+
+ The current container and the project folder container it.
+
+
+ The current container and all of its parent containers.
+
+
+ The current container, its project folder and all shared folders.
+
+
+ Current folder, project, and Shared project.
+
+
+ Current workbook and parent.
+
+
+ Current study and its source/parent study.
+
+
+ All folders to which the user has permission.
+
+
+
+
+
+