Skip to content

Commit

Permalink
fix(build): Fix E2E tests (#1526)
Browse files Browse the repository at this point in the history
  • Loading branch information
Benjamin Geer authored Nov 26, 2019
1 parent cf4f67e commit b3d1638
Show file tree
Hide file tree
Showing 24 changed files with 409 additions and 131 deletions.
48 changes: 33 additions & 15 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,11 @@ jobs:
uses: actions/cache@v1
with:
path: ~/.ivy2
key: ${{ runner.OS }}
key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
restore-keys: |
${{ runner.OS }}
${{ runner.OS }}-build-${{ env.cache-name }}-
${{ runner.OS }}-build-
${{ runner.OS }}-
- name: Run build everything
run: sbt clean test:compile it:compile stage

Expand All @@ -59,7 +61,11 @@ jobs:
uses: actions/cache@v1
with:
path: ~/.ivy2
key: ${{ runner.OS }}
key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
restore-keys: |
${{ runner.OS }}-build-${{ env.cache-name }}-
${{ runner.OS }}-build-
${{ runner.OS }}-
- name: prepare needed graphdb-se files
run: |
echo ${{ secrets.license_encryption_key }} | gpg --quiet --batch --yes --decrypt --passphrase-fd 0 --output $GITHUB_WORKSPACE/ci/secrets.tar $GITHUB_WORKSPACE/ci/secrets.tar.gpg
Expand All @@ -84,9 +90,11 @@ jobs:
uses: actions/cache@v1
with:
path: ~/.ivy2
key: ${{ runner.OS }}
key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
restore-keys: |
${{ runner.OS }}
${{ runner.OS }}-build-${{ env.cache-name }}-
${{ runner.OS }}-build-
${{ runner.OS }}-
- name: prepare needed graphdb-se files
run: |
echo ${{ secrets.license_encryption_key }} | gpg --quiet --batch --yes --decrypt --passphrase-fd 0 --output $GITHUB_WORKSPACE/ci/secrets.tar $GITHUB_WORKSPACE/ci/secrets.tar.gpg
Expand All @@ -111,9 +119,11 @@ jobs:
uses: actions/cache@v1
with:
path: ~/.ivy2
key: ${{ runner.OS }}
key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
restore-keys: |
${{ runner.OS }}
${{ runner.OS }}-build-${{ env.cache-name }}-
${{ runner.OS }}-build-
${{ runner.OS }}-
- name: prepare needed graphdb-se files
run: |
echo ${{ secrets.license_encryption_key }} | gpg --quiet --batch --yes --decrypt --passphrase-fd 0 --output $GITHUB_WORKSPACE/ci/secrets.tar $GITHUB_WORKSPACE/ci/secrets.tar.gpg
Expand All @@ -138,9 +148,11 @@ jobs:
uses: actions/cache@v1
with:
path: ~/.ivy2
key: ${{ runner.OS }}
key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
restore-keys: |
${{ runner.OS }}
${{ runner.OS }}-build-${{ env.cache-name }}-
${{ runner.OS }}-build-
${{ runner.OS }}-
- name: run upgrade tests
run: sbt "upgrade/test"

Expand All @@ -159,9 +171,11 @@ jobs:
uses: actions/cache@v1
with:
path: ~/.ivy2
key: ${{ runner.OS }}
key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
restore-keys: |
${{ runner.OS }}
${{ runner.OS }}-build-${{ env.cache-name }}-
${{ runner.OS }}-build-
${{ runner.OS }}-
- name: install requirements
run: sudo apt-get install expect
- name: build all images
Expand Down Expand Up @@ -208,9 +222,11 @@ jobs:
uses: actions/cache@v1
with:
path: ~/.ivy2
key: ${{ runner.OS }}
key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
restore-keys: |
${{ runner.OS }}
${{ runner.OS }}-build-${{ env.cache-name }}-
${{ runner.OS }}-build-
${{ runner.OS }}-
- name: run docs build test
run: |
mkdir -p $GITHUB_WORKSPACE/docs/target
Expand All @@ -231,9 +247,11 @@ jobs:
uses: actions/cache@v1
with:
path: ~/.ivy2
key: ${{ runner.OS }}
key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
restore-keys: |
${{ runner.OS }}
${{ runner.OS }}-build-${{ env.cache-name }}-
${{ runner.OS }}-build-
${{ runner.OS }}-
- name: build and publish all images to Dockerhub
run: |
echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_USER }} --password-stdin
Expand Down
10 changes: 5 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ stack-without-api-and-sipi: stack-up ## starts the knora-stack without knora-api
docker-compose -f docker/knora.docker-compose.yml stop sipi

.PHONY: unit-tests
unit-tests: stack-without-api init-knora-test-unit ## runs the unit tests (equivalent to 'sbt webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest').
unit-tests: stack-without-api init-db-test-unit ## runs the unit tests (equivalent to 'sbt webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest').
docker run --rm \
-v /tmp:/tmp \
-v $(PWD):/src \
Expand Down Expand Up @@ -227,7 +227,7 @@ unit-tests-with-coverage: stack-without-api ## runs the unit tests (equivalent t
daschswiss/scala-sbt sbt coverage 'webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest' webapi/coverageReport

.PHONY: e2e-tests
e2e-tests: stack-without-api init-knora-test-unit ## runs the e2e tests (equivalent to 'sbt webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest').
e2e-tests: stack-without-api init-db-test-unit ## runs the e2e tests (equivalent to 'sbt webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest').
docker run --rm \
-v /tmp:/tmp \
-v $(PWD):/src \
Expand Down Expand Up @@ -257,10 +257,10 @@ e2e-tests-with-coverage: stack-without-api ## runs the e2e tests (equivalent to
-e KNORA_WEBAPI_CACHE_SERVICE_REDIS_HOST=redis \
-e SBT_OPTS="-Xms2048M -Xmx2048M -Xss6M" \
--network=docker_knora-net \
daschswiss/scala-sbt sbt coverage 'webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest' webapi/coverageReport
daschswiss/scala-sbt sbt coverage 'webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest' webapi/coverageReport

.PHONY: it-tests
it-tests: stack-without-api init-knora-test-unit ## runs the integration tests (equivalent to 'sbt webapi/it').
it-tests: stack-without-api init-db-test-unit ## runs the integration tests (equivalent to 'sbt webapi/it').
docker run --rm \
-v /tmp:/tmp \
-v $(PWD):/src \
Expand All @@ -272,7 +272,7 @@ it-tests: stack-without-api init-knora-test-unit ## runs the integration tests (
-e KNORA_WEBAPI_CACHE_SERVICE_REDIS_HOST=redis \
-e SBT_OPTS="-Xms2048M -Xmx2048M -Xss6M" \
--network=docker_knora-net \
daschswiss/scala-sbt sbt "webapi/it:test"
daschswiss/scala-sbt sbt 'webapi/it:test'

.PHONY: it-tests-with-coverage
it-tests-with-coverage: stack-without-api ## runs the integration tests (equivalent to 'sbt webapi/it:test') with code-coverage reporting.
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ A pull request should include tests and documentation for the changes that were

### Technical

Please use the [knora-user](https://www.maillist.unibas.ch/mailman/listinfo/knora-user) mailing list for technical questions.
Please use the [discuss.dasch.swiss](https://discuss.dasch.swiss) forum for technical questions.

### Administrative

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
owlim:check-for-inconsistencies "true";
owlim:base-URL "http://rdfh.ch/" ;
owlim:repository-type "file-repository" ;
owlim:query-timeout "5" ;
owlim:query-timeout "30" ;
owlim:throw-QueryEvaluationException-on-timeout "true";
owlim:query-limit-results "50000";
]
Expand Down
4 changes: 1 addition & 3 deletions webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,7 @@ class ITKnoraLiveSpec(_system: ActorSystem) extends Core with StartupUtils with
getResponseString(request).parseJson.asJsObject
}



protected def singleAwaitingRequest(request: HttpRequest, duration: Duration = 9999.milliseconds): HttpResponse = {
protected def singleAwaitingRequest(request: HttpRequest, duration: Duration = 15999.milliseconds): HttpResponse = {
val responseFuture = Http().singleRequest(request)
Await.result(responseFuture, duration)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ package org.knora.webapi.routing.admin

import org.knora.webapi.OntologyConstants
import org.knora.webapi.routing.KnoraRouteData
import org.knora.webapi.util.{SmartIri, StringFormatter}
import org.knora.webapi.util.IriConversions._
import org.knora.webapi.util.clientapi._
import org.knora.webapi.util.{SmartIri, StringFormatter}


/**
Expand Down Expand Up @@ -129,14 +129,23 @@ class AdminClientApi(routeData: KnoraRouteData) extends ClientApi {
).map(_.toSmartIri)

/**
* A map of property IRIs to non-standard names that those properties must have.
* A map of class IRIs to maps of property IRIs to non-standard names that those properties must have
* in those classes. Needed only for JSON, and only if two different properties should have the same name in
* different classes. `JsonInstanceInspector` also needs to know about these.
*/
override val propertyNames: Map[SmartIri, String] = Map(
OntologyConstants.KnoraAdminV2.ProjectIri -> "project",
OntologyConstants.KnoraAdminV2.ProjectDescription -> "description",
OntologyConstants.KnoraAdminV2.GroupDescription -> "description"
override lazy val propertyNames: Map[SmartIri, Map[SmartIri, String]] = AdminClientApi.propertyNames
}

object AdminClientApi {
def propertyNames(implicit stringFormatter: StringFormatter): Map[SmartIri, Map[SmartIri, String]] = Map(
OntologyConstants.KnoraAdminV2.CreateGroupRequest -> Map(OntologyConstants.KnoraAdminV2.ProjectIri -> "project"),
OntologyConstants.KnoraAdminV2.ProjectClass -> Map(OntologyConstants.KnoraAdminV2.ProjectDescription -> "description"),
OntologyConstants.KnoraAdminV2.GroupClass -> Map(OntologyConstants.KnoraAdminV2.GroupDescription -> "description")
).map {
case (propertyIri, propertyName) =>
propertyIri.toSmartIri -> propertyName
case (classIri, propertyMap) =>
classIri.toSmartIri -> propertyMap.map {
case (propertyIri, propertyName) =>
propertyIri.toSmartIri -> propertyName
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,11 @@ class V2ClientApi (routeData: KnoraRouteData) extends ClientApi {
override val idProperties: Set[SmartIri] = Set.empty

/**
* A map of property IRIs to non-standard names that those properties must have.
* A map of class IRIs to maps of property IRIs to non-standard names that those properties must have
* in those classes. Needed only for JSON, and only if two different properties should have the same name in
* different classes. `JsonInstanceInspector` also needs to know about these.
*/
override val propertyNames: Map[SmartIri, String] = Map.empty
override val propertyNames: Map[SmartIri, Map[SmartIri, String]] = Map.empty
/**
* A map of class IRIs to IRIs of optional set properties. Such properties have cardinality 0-n, and should
* be made optional in generated code.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1170,7 +1170,7 @@ class StringFormatter private(val maybeSettings: Option[SettingsImpl] = None, ma

override def isKnoraOntologyIri: Boolean = iriInfo.iriType == KnoraDefinitionIri && iriInfo.ontologyName.nonEmpty && iriInfo.entityName.isEmpty

override def isKnoraEntityIri: Boolean = iriInfo.iriType == KnoraDefinitionIri && iriInfo.entityName.nonEmpty
override def isKnoraEntityIri: Boolean = iriInfo.iriType == KnoraDefinitionIri && iriInfo.entityName.nonEmpty && iriInfo.clientCollectionType.isEmpty

override def isClientCollectionTypeIri: Boolean = iriInfo.clientCollectionType.nonEmpty

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,10 +114,11 @@ trait ClientApi {
val idProperties: Set[SmartIri]

/**
* A map of property IRIs to non-standard names that those properties must have.
* Needed only if two different properties should have the same name in different classes.
* A map of class IRIs to maps of property IRIs to non-standard names that those properties must have
* in those classes. Needed only for JSON, and only if two different properties should have the same name in
* different classes. `JsonInstanceInspector` also needs to know about these.
*/
val propertyNames: Map[SmartIri, String]
val propertyNames: Map[SmartIri, Map[SmartIri, String]]

/**
* Class IRIs that are used by this API, other than the ones used in endpoints.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -237,8 +237,13 @@ class GeneratorFrontEnd(routeData: KnoraRouteData, requestingUser: UserADM) {
case (classIri: SmartIri, classDef: ClientClassDefinition) =>
val transformedProps = classDef.properties.map {
propDef =>
clientApi.propertyNames.get(propDef.propertyIri) match {
case Some(propertyName) => propDef.copy(propertyName = propertyName)
clientApi.propertyNames.get(classIri) match {
case Some(propertyMap) =>
propertyMap.get(propDef.propertyIri) match {
case Some(propertyName) => propDef.copy(propertyName = propertyName)
case None => propDef
}

case None => propDef
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,15 +144,15 @@
},
"knora-api:fileValueAsUrl" : {
"@type" : "xsd:anyURI",
"@value" : "http://0.0.0.0:1024/0803/incunabula_0000004159.jp2/full/3625,5251/0/default.jpg"
"@value" : "IIIF_BASE_URL/0803/incunabula_0000004159.jp2/full/3625,5251/0/default.jpg"
},
"knora-api:fileValueHasFilename" : "incunabula_0000004159.jp2",
"knora-api:hasPermissions" : "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser",
"knora-api:stillImageFileValueHasDimX" : 3625,
"knora-api:stillImageFileValueHasDimY" : 5251,
"knora-api:stillImageFileValueHasIIIFBaseUrl" : {
"@type" : "xsd:anyURI",
"@value" : "http://0.0.0.0:1024/0803"
"@value" : "IIIF_BASE_URL/0803"
},
"knora-api:userHasPermission" : "RV",
"knora-api:valueCreationDate" : {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,15 +144,15 @@
},
"http://api.knora.org/ontology/knora-api/v2#fileValueAsUrl": {
"@type": "http://www.w3.org/2001/XMLSchema#anyURI",
"@value": "http://0.0.0.0:1024/0803/incunabula_0000004159.jp2/full/3625,5251/0/default.jpg"
"@value": "IIIF_BASE_URL/0803/incunabula_0000004159.jp2/full/3625,5251/0/default.jpg"
},
"http://api.knora.org/ontology/knora-api/v2#fileValueHasFilename": "incunabula_0000004159.jp2",
"http://api.knora.org/ontology/knora-api/v2#hasPermissions": "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser",
"http://api.knora.org/ontology/knora-api/v2#stillImageFileValueHasDimX": 3625,
"http://api.knora.org/ontology/knora-api/v2#stillImageFileValueHasDimY": 5251,
"http://api.knora.org/ontology/knora-api/v2#stillImageFileValueHasIIIFBaseUrl": {
"@type": "http://www.w3.org/2001/XMLSchema#anyURI",
"@value": "http://0.0.0.0:1024/0803"
"@value": "IIIF_BASE_URL/0803"
},
"http://api.knora.org/ontology/knora-api/v2#userHasPermission": "RV",
"http://api.knora.org/ontology/knora-api/v2#valueCreationDate": {
Expand Down
46 changes: 46 additions & 0 deletions webapi/src/test/resources/test-data/resourcesR2RV2/Testding.jsonld
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,52 @@
"@value" : "http://0.0.0.0:3336/ark:/72163/1/0001/H6gBWUuJSuuO=CilHV8kQwk/bXMwnrHvQH2DMjOFrGmNzgz.20180528T155203897Z"
}
},
"anything:hasGeometry" : {
"@id" : "http://rdfh.ch/0001/H6gBWUuJSuuO-CilHV8kQw/values/we-ybmj-SRen-91n4RaDOQ",
"@type" : "knora-api:GeomValue",
"knora-api:arkUrl" : {
"@type" : "xsd:anyURI",
"@value" : "http://0.0.0.0:3336/ark:/72163/1/0001/H6gBWUuJSuuO=CilHV8kQwk/we=ybmj=SRen=91n4RaDOQB"
},
"knora-api:attachedToUser" : {
"@id" : "http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ"
},
"knora-api:geometryValueAsGeometry" : "{\"status\":\"active\",\"lineColor\":\"#ff3333\",\"lineWidth\":2,\"points\":[{\"x\":0.08098591549295775,\"y\":0.16741071428571427},{\"x\":0.7394366197183099,\"y\":0.7299107142857143}],\"type\":\"rectangle\",\"original_index\":0}",
"knora-api:hasPermissions" : "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser",
"knora-api:userHasPermission" : "RV",
"knora-api:valueCreationDate" : {
"@type" : "xsd:dateTimeStamp",
"@value" : "2018-05-28T15:52:03.897Z"
},
"knora-api:valueHasUUID" : "we-ybmj-SRen-91n4RaDOQ",
"knora-api:versionArkUrl" : {
"@type" : "xsd:anyURI",
"@value" : "http://0.0.0.0:3336/ark:/72163/1/0001/H6gBWUuJSuuO=CilHV8kQwk/we=ybmj=SRen=91n4RaDOQB.20180528T155203897Z"
}
},
"anything:hasGeoname" : {
"@id" : "http://rdfh.ch/0001/H6gBWUuJSuuO-CilHV8kQw/values/hty-ONF8SwKN2RKU7rLKDg",
"@type" : "knora-api:GeonameValue",
"knora-api:arkUrl" : {
"@type" : "xsd:anyURI",
"@value" : "http://0.0.0.0:3336/ark:/72163/1/0001/H6gBWUuJSuuO=CilHV8kQwk/hty=ONF8SwKN2RKU7rLKDgn"
},
"knora-api:attachedToUser" : {
"@id" : "http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ"
},
"knora-api:geonameValueAsGeonameCode" : "2661604",
"knora-api:hasPermissions" : "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser",
"knora-api:userHasPermission" : "RV",
"knora-api:valueCreationDate" : {
"@type" : "xsd:dateTimeStamp",
"@value" : "2018-05-28T15:52:03.897Z"
},
"knora-api:valueHasUUID" : "hty-ONF8SwKN2RKU7rLKDg",
"knora-api:versionArkUrl" : {
"@type" : "xsd:anyURI",
"@value" : "http://0.0.0.0:3336/ark:/72163/1/0001/H6gBWUuJSuuO=CilHV8kQwk/hty=ONF8SwKN2RKU7rLKDgn.20180528T155203897Z"
}
},
"anything:hasInteger" : {
"@id" : "http://rdfh.ch/0001/H6gBWUuJSuuO-CilHV8kQw/values/dJ1ES8QTQNepFKF5-EAqdg",
"@type" : "knora-api:IntValue",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,15 +149,15 @@
},
"knora-api:fileValueAsUrl" : {
"@type" : "xsd:anyURI",
"@value" : "http://0.0.0.0:1024/0803/incunabula_0000003856.jp2/full/1904,2700/0/default.jpg"
"@value" : "IIIF_BASE_URL/0803/incunabula_0000003856.jp2/full/1904,2700/0/default.jpg"
},
"knora-api:fileValueHasFilename" : "incunabula_0000003856.jp2",
"knora-api:hasPermissions" : "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser",
"knora-api:stillImageFileValueHasDimX" : 1904,
"knora-api:stillImageFileValueHasDimY" : 2700,
"knora-api:stillImageFileValueHasIIIFBaseUrl" : {
"@type" : "xsd:anyURI",
"@value" : "http://0.0.0.0:1024/0803"
"@value" : "IIIF_BASE_URL/0803"
},
"knora-api:userHasPermission" : "RV",
"knora-api:valueCreationDate" : {
Expand Down
4 changes: 2 additions & 2 deletions webapi/src/test/scala/org/knora/webapi/E2ESpec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -193,10 +193,10 @@ class E2ESpec(_system: ActorSystem) extends Core with StartupUtils with Triplest
*/
protected def readOrWriteTextFile(responseAsString: String, file: File, writeFile: Boolean = false): String = {
if (writeFile) {
FileUtil.writeTextFile(file, responseAsString)
FileUtil.writeTextFile(file, responseAsString.replaceAll(settings.externalSipiIIIFGetUrl, "IIIF_BASE_URL"))
responseAsString
} else {
FileUtil.readTextFile(file)
FileUtil.readTextFile(file).replaceAll("IIIF_BASE_URL", settings.externalSipiIIIFGetUrl)
}
}
}
Loading

0 comments on commit b3d1638

Please sign in to comment.