From 3a61c57e714ab281e0438a0e51761a8b3a68795c Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 21 Dec 2018 15:45:07 +0200 Subject: [PATCH 01/70] NY-6318 Update file metadata - add enum providers; Signed-off-by: abotev-intracol --- .../content/file/metadata/FileMetadataService.java | 5 +++-- .../content/file/storage/StorageProviderPool.java | 8 ++++---- .../configuration/ContentServiceConfiguration.java | 13 ++++++++++--- .../content/grpc/services/ContentServiceImpl.java | 6 ++++-- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index 483d9fe..c84e17f 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.metadata.repositories.FileMetadataReposiory; +import biz.nynja.content.file.storage.Providers; import biz.nynja.content.upload.models.UploadStore; /** @@ -29,7 +30,7 @@ public class FileMetadataService { this.fileMetadataRepository = fileMetadataRepository; } - public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl) { + public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl, Providers storageProvider) { FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setKey(UUID.randomUUID()); fileMetadata.setFileName(fileName); @@ -39,7 +40,7 @@ public class FileMetadataService { fileMetadata.setUploadTimestamp(Instant.now().toEpochMilli()); fileMetadata.setAccountId(uploadInfo.getAccountId()); fileMetadata.setDeviceId(uploadInfo.getDeviceId()); - fileMetadata.setStorage("LOCAL"); + fileMetadata.setStorage(storageProvider.name()); fileMetadata.setFileUrl(fileUrl); fileMetadataRepository.save(fileMetadata); diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java index 2a2b310..76cb33f 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java @@ -23,14 +23,14 @@ public class StorageProviderPool { private static final Logger log = LoggerFactory.getLogger(StorageProviderPool.class); - private Map storageProviders = new HashMap<>(); + private Map storageProviders = new HashMap<>(); public StorageProviderPool(GoogleStorageProvider googleStorageProvider, LocalStorageProvider localStorageProvider) { - storageProviders.put("LOCAL", localStorageProvider); - storageProviders.put("GOOGLE", googleStorageProvider); + storageProviders.put(Providers.LOCAL, localStorageProvider); + storageProviders.put(Providers.GOOGLE, googleStorageProvider); } - public StorageProvider getStorageProviderByType(String storageProviderType) { + public StorageProvider getStorageProviderByType(Providers storageProviderType) { log.debug("Get storage provider = {}", storageProviderType); StorageProvider storageProvider = storageProviders.get(storageProviderType); if (storageProvider == null) { diff --git a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java index 4d81a0a..27f9686 100644 --- a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java +++ b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java @@ -7,6 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; +import biz.nynja.content.file.storage.Providers; + /** * @author Ralitsa Todorova * @@ -16,13 +18,18 @@ public class ContentServiceConfiguration { private final String uploadUrl; private final String downloadUrl; - private final String storageProvider; + private final Providers storageProvider; @Autowired public ContentServiceConfiguration(Environment env) { this.uploadUrl = env.getRequiredProperty("file.upload.url"); this.downloadUrl = env.getRequiredProperty("file.download.url"); - this.storageProvider = env.getRequiredProperty("storage.provider"); + this.storageProvider = getProvidersProperty("storage.provider", env); + } + + private Providers getProvidersProperty(String key, Environment env) { + String provider = env.getRequiredProperty("storage.provider"); + return Providers.valueOf(provider); } public String getUploadUrl() { @@ -33,7 +40,7 @@ public class ContentServiceConfiguration { return downloadUrl; } - public String getStorageProvider() { + public Providers getStorageProvider() { return storageProvider; } } diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 722efed..9b4402e 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -19,6 +19,7 @@ import org.springframework.util.StringUtils; import biz.nynja.content.core.validation.Validation; import biz.nynja.content.file.metadata.FileMetadataService; import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.Providers; import biz.nynja.content.file.storage.StorageProvider; import biz.nynja.content.file.storage.StorageProviderPool; import biz.nynja.content.grpc.ContentServiceGrpc; @@ -148,6 +149,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas private int receivedBytesCount; private String fileLocation; private int reachedSize; + private Providers provider = contentSreviceConfiguration.getStorageProvider(); public UploadStreamObserver(StreamObserver responseObserver) { this.responseObserver = responseObserver; @@ -168,7 +170,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas try { if (storageProvider == null) { storageProvider = storageProviderPool - .getStorageProviderByType(contentSreviceConfiguration.getStorageProvider()); + .getStorageProviderByType(provider); fileName = constructFileName(uploadInfo.getFileName()); fileLocation = storageProvider.initialize(fileName); uploadInfo.setStatus("IN_PROGRESS"); @@ -213,7 +215,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas Status.OUT_OF_RANGE.withDescription("Uploaded file size does not meet expectations."))); return; } - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, fileLocation); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, fileLocation, provider); responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); -- GitLab From 4f20cf50c6f5a23c9969ee22b43e2d2c11753d7f Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 11 Jan 2019 17:26:03 +0200 Subject: [PATCH 02/70] NY-6318 Update file metadata - add storage providers; - add file download url generate; - update configuration files; Signed-off-by: abotev-intracol --- .../biz/nynja/content/file/storage/Providers.java | 12 ++++++++++++ .../nynja/content/file/storage/StorageProvider.java | 1 + .../file/storage/impl/GoogleStorageProvider.java | 8 ++++++++ .../file/storage/impl/LocalStorageProvider.java | 5 +++++ .../content/grpc/services/ContentServiceImpl.java | 2 +- src/main/resources/application-dev.yml | 2 +- src/main/resources/application-production.yml | 2 +- 7 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 src/main/java/biz/nynja/content/file/storage/Providers.java diff --git a/src/main/java/biz/nynja/content/file/storage/Providers.java b/src/main/java/biz/nynja/content/file/storage/Providers.java new file mode 100644 index 0000000..55964cc --- /dev/null +++ b/src/main/java/biz/nynja/content/file/storage/Providers.java @@ -0,0 +1,12 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.storage; + +/** + * @author Angel.Botev + * + */ +public enum Providers { + GOOGLE, LOCAL +} diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index 22b6990..cc4df34 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -15,4 +15,5 @@ public interface StorageProvider { public void close(String fileLocation); + public String getFileUrl(String filename); } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index f677152..785df48 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -160,4 +160,12 @@ public class GoogleStorageProvider implements StorageProvider { HttpResponse resp = req.execute(); return resp; } + + @Override + public String getFileUrl(String filename) { + StringBuilder builder = new StringBuilder(); + builder.append(storageConfiguration.getGoogleStorageURI()).append("/") + .append(storageConfiguration.getGoogleBucketName()).append("/").append(filename); + return builder.toString(); + } } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index b9df377..71573d4 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -58,4 +58,9 @@ public class LocalStorageProvider implements StorageProvider { return new StringBuilder(contentStoreLocation).append(File.separator).append(fileName).toString(); } + @Override + public String getFileUrl(String filename) { + return constructFilePath(storageConfiguration.getLocalStorageLocation(), filename); + } + } diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 9b4402e..10295d1 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -215,7 +215,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas Status.OUT_OF_RANGE.withDescription("Uploaded file size does not meet expectations."))); return; } - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, fileLocation, provider); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, storageProvider.getFileUrl(fileName), provider); responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index bf84675..a54500f 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -42,7 +42,7 @@ file: storage: provider: GOOGLE local: - location: src/main/resources/ + location: src/main/resources google: uri: https://storage.googleapis.com bucket: content-service-dev diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 5418080..a87b0e9 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -42,7 +42,7 @@ file: storage: provider: GOOGLE local: - location: ${LOCAL_STORAGE_LOCATION:/src/main/resources/} + location: ${LOCAL_STORAGE_LOCATION:/src/main/resources} google: uri: ${GOOGLE_STORAGE_URI:https://storage.googleapis.com} bucket: ${GOOGLE_STORAGE_BUCKET:content-service-dev} -- GitLab From 11ba2db54c2dc1e8df423c383de3b9c8c1438c8f Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Mon, 14 Jan 2019 14:51:44 +0200 Subject: [PATCH 03/70] NY-6318 Update file metadata - fix comments; Signed-off-by: abotev-intracol --- .../content/file/metadata/FileMetadataService.java | 4 ++-- .../file/storage/{Providers.java => Provider.java} | 2 +- .../content/file/storage/StorageProviderPool.java | 8 ++++---- .../configuration/ContentServiceConfiguration.java | 12 ++++++------ .../content/grpc/services/ContentServiceImpl.java | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) rename src/main/java/biz/nynja/content/file/storage/{Providers.java => Provider.java} (86%) diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index c84e17f..981dff1 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.metadata.repositories.FileMetadataReposiory; -import biz.nynja.content.file.storage.Providers; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.upload.models.UploadStore; /** @@ -30,7 +30,7 @@ public class FileMetadataService { this.fileMetadataRepository = fileMetadataRepository; } - public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl, Providers storageProvider) { + public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl, Provider storageProvider) { FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setKey(UUID.randomUUID()); fileMetadata.setFileName(fileName); diff --git a/src/main/java/biz/nynja/content/file/storage/Providers.java b/src/main/java/biz/nynja/content/file/storage/Provider.java similarity index 86% rename from src/main/java/biz/nynja/content/file/storage/Providers.java rename to src/main/java/biz/nynja/content/file/storage/Provider.java index 55964cc..3435845 100644 --- a/src/main/java/biz/nynja/content/file/storage/Providers.java +++ b/src/main/java/biz/nynja/content/file/storage/Provider.java @@ -7,6 +7,6 @@ package biz.nynja.content.file.storage; * @author Angel.Botev * */ -public enum Providers { +public enum Provider { GOOGLE, LOCAL } diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java index 76cb33f..461e034 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java @@ -23,14 +23,14 @@ public class StorageProviderPool { private static final Logger log = LoggerFactory.getLogger(StorageProviderPool.class); - private Map storageProviders = new HashMap<>(); + private Map storageProviders = new HashMap<>(); public StorageProviderPool(GoogleStorageProvider googleStorageProvider, LocalStorageProvider localStorageProvider) { - storageProviders.put(Providers.LOCAL, localStorageProvider); - storageProviders.put(Providers.GOOGLE, googleStorageProvider); + storageProviders.put(Provider.LOCAL, localStorageProvider); + storageProviders.put(Provider.GOOGLE, googleStorageProvider); } - public StorageProvider getStorageProviderByType(Providers storageProviderType) { + public StorageProvider getStorageProviderByType(Provider storageProviderType) { log.debug("Get storage provider = {}", storageProviderType); StorageProvider storageProvider = storageProviders.get(storageProviderType); if (storageProvider == null) { diff --git a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java index 27f9686..e520288 100644 --- a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java +++ b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java @@ -7,7 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; -import biz.nynja.content.file.storage.Providers; +import biz.nynja.content.file.storage.Provider; /** * @author Ralitsa Todorova @@ -18,7 +18,7 @@ public class ContentServiceConfiguration { private final String uploadUrl; private final String downloadUrl; - private final Providers storageProvider; + private final Provider storageProvider; @Autowired public ContentServiceConfiguration(Environment env) { @@ -27,9 +27,9 @@ public class ContentServiceConfiguration { this.storageProvider = getProvidersProperty("storage.provider", env); } - private Providers getProvidersProperty(String key, Environment env) { - String provider = env.getRequiredProperty("storage.provider"); - return Providers.valueOf(provider); + private Provider getProvidersProperty(String key, Environment env) { + String provider = env.getRequiredProperty(key); + return Provider.valueOf(provider); } public String getUploadUrl() { @@ -40,7 +40,7 @@ public class ContentServiceConfiguration { return downloadUrl; } - public Providers getStorageProvider() { + public Provider getStorageProvider() { return storageProvider; } } diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 10295d1..1e4f7db 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -19,7 +19,7 @@ import org.springframework.util.StringUtils; import biz.nynja.content.core.validation.Validation; import biz.nynja.content.file.metadata.FileMetadataService; import biz.nynja.content.file.metadata.dto.FileMetadata; -import biz.nynja.content.file.storage.Providers; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageProvider; import biz.nynja.content.file.storage.StorageProviderPool; import biz.nynja.content.grpc.ContentServiceGrpc; @@ -149,7 +149,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas private int receivedBytesCount; private String fileLocation; private int reachedSize; - private Providers provider = contentSreviceConfiguration.getStorageProvider(); + private Provider provider = contentSreviceConfiguration.getStorageProvider(); public UploadStreamObserver(StreamObserver responseObserver) { this.responseObserver = responseObserver; -- GitLab From 72c4b9cf4b6336ae5c8124aef55e15e4b0adf60b Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 15 Jan 2019 14:57:40 +0200 Subject: [PATCH 04/70] NY-6494 Extract keyspace replica factor - add replication configuration and add property in CassandraConfig; Signed-off-by: abotev-intracol --- .../nynja/content/db/configuration/CassandraConfig.java | 9 ++++++++- src/main/resources/application-dev.yml | 1 + src/main/resources/application-production.yml | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java index 525af77..fd051ff 100644 --- a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java +++ b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java @@ -22,6 +22,9 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Value("${spring.data.cassandra.keyspace-name}") private String keyspace; + @Value("${spring.data.cassandra.replication}") + private int replication; + @Override protected String getKeyspaceName() { return keyspace; @@ -43,6 +46,10 @@ public class CassandraConfig extends AbstractCassandraConfiguration { return port; } + private int getReplication() { + return replication; + } + @Override public SchemaAction getSchemaAction() { return SchemaAction.CREATE_IF_NOT_EXISTS; @@ -51,7 +58,7 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Override protected List getKeyspaceCreations() { CreateKeyspaceSpecification specification = CreateKeyspaceSpecification.createKeyspace(getKeyspaceName()) - .ifNotExists().withSimpleReplication(); + .ifNotExists().withSimpleReplication(getReplication()); return Arrays.asList(specification); } diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index a54500f..6da8f26 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -15,6 +15,7 @@ spring: keyspace-name: content contact-points: localhost port: 9042 + replication: 3 token: encryptdecrypt: diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index a87b0e9..b2e6ee8 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -15,6 +15,7 @@ spring: keyspace-name: ${CASSANDRA_KEYSPACE:content} contact-points: ${CASSANDRA_CONTACT_POINTS:cassandra.cassandra.svc.cluster.local} port: ${CASSANDRA_PORT:9042} + replication: ${CASSANDRA_KEYSPACE_REPLICATION:3} token: encryptdecrypt: -- GitLab From 6f60487da7c0f00bd8d54107e93c2ed115fa5e54 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 16 Jan 2019 18:14:09 +0200 Subject: [PATCH 05/70] NY-6725 Remove unnecessary logger - replace Groovy library with Spring default; Signed-off-by: abotev-intracol --- pom.xml | 24 ++++++++++---- src/main/resources/application-dev.yml | 32 +++++++++++++++++++ src/main/resources/application-production.yml | 32 +++++++++++++++++++ src/main/resources/logback-spring.groovy | 32 ------------------- 4 files changed, 81 insertions(+), 39 deletions(-) delete mode 100644 src/main/resources/logback-spring.groovy diff --git a/pom.xml b/pom.xml index 15468fb..bc4e4ea 100644 --- a/pom.xml +++ b/pom.xml @@ -124,8 +124,18 @@ - org.codehaus.groovy - groovy-all + org.springframework.boot + spring-boot-starter-actuator + + + + io.micrometer + micrometer-core + + + + io.micrometer + micrometer-registry-prometheus @@ -152,11 +162,11 @@ true - - com.google.cloud - google-cloud-storage - 1.53.0 - + + com.google.cloud + google-cloud-storage + 1.53.0 + diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 6da8f26..4fc172e 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -48,3 +48,35 @@ storage: uri: https://storage.googleapis.com bucket: content-service-dev upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + +# To enable colors in Eclipse: +# spring.output.ansi.enabled=ALWAYS and in eclipse +# Help -> Install New Software... and click "Add..." to add the following URL: +# http://www.mihai-nita.net/eclipse + output: + ansi: + enabled: ALWAYS + +logging: + level: + root: INFO + org: + springframework: + web: INFO + +#Metrics related configurations +management: + endpoint: + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + exposure: + include: 'prometheus, health, info, loggers' + metrics: + export: + prometheus: + enabled: true + diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index b2e6ee8..63f2d69 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -48,3 +48,35 @@ storage: uri: ${GOOGLE_STORAGE_URI:https://storage.googleapis.com} bucket: ${GOOGLE_STORAGE_BUCKET:content-service-dev} upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + +# To enable colors in Eclipse: +# spring.output.ansi.enabled=ALWAYS and in eclipse +# Help -> Install New Software... and click "Add..." to add the following URL: +# http://www.mihai-nita.net/eclipse + output: + ansi: + enabled: ALWAYS + +logging: + level: + root: INFO + org: + springframework: + web: INFO + +#Metrics related configurations +management: + endpoint: + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + exposure: + include: 'prometheus, health, info, loggers' + metrics: + export: + prometheus: + enabled: true + diff --git a/src/main/resources/logback-spring.groovy b/src/main/resources/logback-spring.groovy deleted file mode 100644 index 91261d1..0000000 --- a/src/main/resources/logback-spring.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ - -import ch.qos.logback.classic.encoder.PatternLayoutEncoder -import ch.qos.logback.core.rolling.RollingFileAppender -import ch.qos.logback.core.rolling.TimeBasedRollingPolicy -import ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP -import ch.qos.logback.core.status.OnConsoleStatusListener -import ch.qos.logback.classic.Level - -statusListener(OnConsoleStatusListener) - -def file = "${System.getProperty('log.dir', '')}content-service-%d.%i.log" - -appender("FILE", RollingFileAppender) { - // add a status message regarding the file property - addInfo("Starting logging to $file") - append = true - encoder(PatternLayoutEncoder) { pattern = "%d{HH:mm:ss.SSS} %level %logger - %msg%n" } - rollingPolicy(TimeBasedRollingPolicy) { - fileNamePattern = "$file" - timeBasedFileNamingAndTriggeringPolicy(SizeAndTimeBasedFNATP) { maxFileSize = "10mb" }} -} - -appender("Console-Appender", ConsoleAppender) { - encoder(PatternLayoutEncoder) { pattern = "%d{HH:mm:ss.SSS} %level %logger - %msg%n" } -} - -logger("org.springframework.cloud.config.client.ConfigServicePropertySourceLocator", Level.WARN) -root(INFO, ["Console-Appender"]) -root(Level.toLevel("${System.getProperty('log.level', 'INFO')}"), ["FILE"]) \ No newline at end of file -- GitLab From 8f10ee8614a24bb87769e8c147ecc4199eb2948b Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Thu, 17 Jan 2019 16:21:31 +0200 Subject: [PATCH 06/70] Add maven skipTests=true property. --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cc232c3..9663d5d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } } } @@ -83,7 +83,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } dockerBuildAndPushToRegistry "${NAMESPACE}/${APP_NAME}", [IMAGE_BUILD_TAG] } -- GitLab From 63b5b5d98bced50c7b7880fe5b5dac77c75032b9 Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Fri, 18 Jan 2019 14:06:25 +0200 Subject: [PATCH 07/70] Add skipTests-true property for master and pr-* branches --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9663d5d..cfa5122 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,7 +44,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } } } @@ -120,7 +120,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } dockerBuildAndPushToRegistry "${NAMESPACE}/${APP_NAME}", [IMAGE_BUILD_TAG] } -- GitLab From 1f8282a814c204da6fab485769bf3156d1c5fc18 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Thu, 17 Jan 2019 19:23:56 +0200 Subject: [PATCH 08/70] Change request mapping name in download controller Signed-off-by: Ralitsa Todorova --- .../nynja/content/file/download/FileDownloadController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java index 1af8123..5933218 100644 --- a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java +++ b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java @@ -32,7 +32,7 @@ import com.nimbusds.jose.util.Base64URL; */ @RestController -@RequestMapping("file/") +@RequestMapping("file/download") public class FileDownloadController { private static final Logger logger = LoggerFactory.getLogger(FileDownloadController.class); @@ -44,7 +44,7 @@ public class FileDownloadController { this.fileDownloadService = fileDownloadService; } - @RequestMapping(method = RequestMethod.GET, value = "download/{fileKey}") + @RequestMapping(method = RequestMethod.GET, value = "/{fileKey}") public ResponseEntity downloadFile(@PathVariable String fileKey, HttpServletRequest request) throws FileNotFoundException { logger.info("Downloading file: {} ...", fileKey); -- GitLab From 28b5aef70e7ff3d812738c8926888155efd45d73 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Thu, 17 Jan 2019 19:27:04 +0200 Subject: [PATCH 09/70] NY-6474: Introduce UploadStatus enum Signed-off-by: Ralitsa Todorova --- .../content/grpc/services/ContentServiceImpl.java | 10 ++++++---- .../nynja/content/upload/models/UploadStatus.java | 13 +++++++++++++ .../nynja/content/upload/models/UploadStore.java | 11 ++++------- 3 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 src/main/java/biz/nynja/content/upload/models/UploadStatus.java diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 1e4f7db..629a518 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -31,6 +31,7 @@ import biz.nynja.content.grpc.UploadTokenResponse; import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; import biz.nynja.content.grpc.constants.ContentServiceConstants; import biz.nynja.content.grpc.interceptors.UploadInterceptor; +import biz.nynja.content.upload.models.UploadStatus; import biz.nynja.content.upload.models.UploadStore; import biz.nynja.content.upload.token.UploadToken; import biz.nynja.content.upload.token.UploadTokenResponseProvider; @@ -109,11 +110,12 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas logger.error("Error generating upload token:{} ", token.toString()); return; } + UUID jobId = UUID.randomUUID(); UploadStore uploadInfo = new UploadStore(tokenResponse.get(), Instant.now().toEpochMilli() + uploadTokenConfiguration.getTokenTimeToLive(), - contentSreviceConfiguration.getUploadUrl(), UUID.randomUUID(), request.getDeviceId(), + contentSreviceConfiguration.getUploadUrl() + jobId, jobId, request.getDeviceId(), UUID.fromString(accountId), request.getName(), request.getSize(), request.getMediaType().name(), - request.getUploadedFrom(), "PENDING"); + request.getUploadedFrom(), UploadStatus.PENDING); if (!uploadTokenService.storeToken(uploadInfo, false)) { uploadTokenResponseProvider.prepareErrorResponse(responseObserver, Cause.INTERNAL_SERVER_ERROR, @@ -173,7 +175,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas .getStorageProviderByType(provider); fileName = constructFileName(uploadInfo.getFileName()); fileLocation = storageProvider.initialize(fileName); - uploadInfo.setStatus("IN_PROGRESS"); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); if (!uploadTokenService.storeToken(uploadInfo, true)) { handleStreamTermination(); responseObserver.onError(new StatusException( @@ -226,7 +228,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas private void handleStreamTermination() { uploadInfo.setRetries(uploadInfo.getRetries() + 1); - uploadInfo.setStatus("FAILED"); + uploadInfo.setStatus(UploadStatus.FAILED); uploadTokenService.storeToken(uploadInfo, false); storageProvider.close(fileLocation); } diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java b/src/main/java/biz/nynja/content/upload/models/UploadStatus.java new file mode 100644 index 0000000..d0380b5 --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/models/UploadStatus.java @@ -0,0 +1,13 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.models; + +/** + * @author Ralitsa Todorova + * + */ +public enum UploadStatus { + + PENDING, IN_PROGRESS, FAILED; +} diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/upload/models/UploadStore.java index 0cc0756..4df158f 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/upload/models/UploadStore.java @@ -26,7 +26,7 @@ public class UploadStore { private int fileSize; private String mediaType; private String uploadedFrom; - private String status; + private UploadStatus status; private int retries; public UploadStore() { @@ -116,11 +116,11 @@ public class UploadStore { this.uploadedFrom = uploadedFrom; } - public String getStatus() { + public UploadStatus getStatus() { return status; } - public void setStatus(String status) { + public void setStatus(UploadStatus status) { this.status = status; } @@ -201,10 +201,7 @@ public class UploadStore { return false; if (retries != other.retries) return false; - if (status == null) { - if (other.status != null) - return false; - } else if (!status.equals(other.status)) + if (status != other.status) return false; if (tokenExpirationTime == null) { if (other.tokenExpirationTime != null) -- GitLab From 610ea5b31e0ff6d380b495b453486221092821b8 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Thu, 17 Jan 2019 19:29:28 +0200 Subject: [PATCH 10/70] NY-6474: Extend UploadStore model - added partId and lastUploadedByte fields Signed-off-by: Ralitsa Todorova --- .../content/upload/models/UploadStore.java | 31 +++++++++++++++++-- .../CustomizedUploadStoreRepositoryImpl.java | 9 +++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/upload/models/UploadStore.java index 4df158f..ec9c10a 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/upload/models/UploadStore.java @@ -28,12 +28,14 @@ public class UploadStore { private String uploadedFrom; private UploadStatus status; private int retries; + private int partId; + private int lastUploadedByte; public UploadStore() { } public UploadStore(String uploadToken, Long tokenExpirationTime, String uploadUrl, UUID jobId, String deviceId, - UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, String status) { + UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, UploadStatus status) { this.uploadToken = uploadToken; this.tokenExpirationTime = tokenExpirationTime; this.uploadUrl = uploadUrl; @@ -46,6 +48,8 @@ public class UploadStore { this.uploadedFrom = uploadedFrom; this.status = status; this.retries = 0; + this.partId = 0; + this.lastUploadedByte = 0; } public String getUploadToken() { @@ -144,6 +148,22 @@ public class UploadStore { this.retries = retries; } + public int getPartId() { + return partId; + } + + public void setPartId(int partId) { + this.partId = partId; + } + + public int getLastUploadedByte() { + return lastUploadedByte; + } + + public void setLastUploadedByte(int lastUploadedByte) { + this.lastUploadedByte = lastUploadedByte; + } + @Override public int hashCode() { final int prime = 31; @@ -153,7 +173,9 @@ public class UploadStore { result = prime * result + ((fileName == null) ? 0 : fileName.hashCode()); result = prime * result + fileSize; result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); + result = prime * result + lastUploadedByte; result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); + result = prime * result + partId; result = prime * result + retries; result = prime * result + ((status == null) ? 0 : status.hashCode()); result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); @@ -194,11 +216,15 @@ public class UploadStore { return false; } else if (!jobId.equals(other.jobId)) return false; + if (lastUploadedByte != other.lastUploadedByte) + return false; if (mediaType == null) { if (other.mediaType != null) return false; } else if (!mediaType.equals(other.mediaType)) return false; + if (partId != other.partId) + return false; if (retries != other.retries) return false; if (status != other.status) @@ -233,7 +259,8 @@ public class UploadStore { .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") - .append(retries).append("]").toString(); + .append(retries).append(", partId=").append(partId).append(", lastUploadedByte=") + .append(lastUploadedByte).append("]").toString(); } } diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java b/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java index c2caaaa..e82580e 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java +++ b/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java @@ -35,13 +35,14 @@ public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadS } private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { - String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + ttl + ";"; return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), - uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), uploadStore.getStatus(), - uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), uploadStore.getUploadUrl(), - uploadStore.getRetries()); + uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), + uploadStore.getStatus().toString(), uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), + uploadStore.getUploadUrl(), uploadStore.getRetries(), uploadStore.getPartId(), + uploadStore.getLastUploadedByte()); } } -- GitLab From 27db02f42ffda22a382702d016b88af380ea55d9 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Thu, 17 Jan 2019 19:32:47 +0200 Subject: [PATCH 11/70] NY-6474: Implement REST endpoint for resumable file upload on chunks Signed-off-by: Ralitsa Todorova --- .../storage/impl/LocalStorageProvider.java | 8 +- .../upload/rest/FileUploadController.java | 157 ++++++++++++++++++ .../rest/validation/ChunkUploadValidator.java | 74 +++++++++ .../rest/validation/ValidationResult.java | 26 +++ src/main/resources/application-dev.yml | 2 +- src/main/resources/application-production.yml | 2 +- 6 files changed, 263 insertions(+), 6 deletions(-) create mode 100644 src/main/java/biz/nynja/content/upload/rest/FileUploadController.java create mode 100644 src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java create mode 100644 src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 71573d4..0aa9656 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -40,13 +40,13 @@ public class LocalStorageProvider implements StorageProvider { @Override public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws IOException { - logger.debug("Appending bytes in file: {}", fileLocation); + logger.debug("Appending bytes to file: {}", fileLocation); FileOutputStream output = new FileOutputStream(fileLocation, true); try { - output.write(data); - logger.info("Bytes was successfully append to file {}.", fileLocation); + output.write(data); + logger.info("Bytes were successfully append to file {}.", fileLocation); } finally { - output.close(); + output.close(); } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java new file mode 100644 index 0000000..40894bf --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -0,0 +1,157 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest; + +import java.util.Optional; + +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import biz.nynja.content.file.metadata.FileMetadataService; +import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.storage.StorageProviderPool; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; +import biz.nynja.content.upload.models.UploadStatus; +import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.upload.rest.validation.ValidationResult; +import biz.nynja.content.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@RestController +@RequestMapping("/file/upload") +public class FileUploadController { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); + + private StorageProvider storageProvider; + private StorageProviderPool storageProviderPool; + private ContentServiceConfiguration contentServiceConfiguration; + private UploadTokenService uploadTokenService; + private FileMetadataService fileMetadataService; + private ChunkUploadValidator validator; + + public FileUploadController(StorageProviderPool storageProviderPool, + ContentServiceConfiguration contentServiceConfiguration, UploadTokenService uploadTokenService, + FileMetadataService fileMetadataService, ChunkUploadValidator validator) { + this.storageProviderPool = storageProviderPool; + this.contentServiceConfiguration = contentServiceConfiguration; + this.uploadTokenService = uploadTokenService; + this.fileMetadataService = fileMetadataService; + this.validator = validator; + } + + @PutMapping("/{jobId}") + public ResponseEntity uploadFile(@PathVariable String jobId, @RequestParam("token") String uploadToken, + @RequestParam("partId") int partId, @RequestParam("last") boolean last, HttpServletRequest request, + @RequestBody byte[] dataChunk) { + + logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); + UploadStore uploadInfo; + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (uploadInfoResult.isPresent()) { + uploadInfo = uploadInfoResult.get(); + } else { + logger.error("You are not allowed to upload data. Provided token is either wrong or missing."); + return new ResponseEntity<>( + "You are not allowed to upload data. Provided token is either wrong or missing.", + HttpStatus.BAD_REQUEST); + } + + Optional requestValidation = validator.validateUploadRequest(uploadInfo, dataChunk, jobId, + uploadToken,partId, last); + if (requestValidation.isPresent()) { + return new ResponseEntity<>(requestValidation.get().getMessage(), HttpStatus.BAD_REQUEST); + } + + String fileName = retrieveFileName(uploadInfo); + String fileLocation; + + try { + if (storageProvider == null) { + storageProvider = storageProviderPool + .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); + } + + fileLocation = storageProvider.initialize(fileName); + logger.info("Created output stream for file {}", fileLocation); + + storageProvider.write(dataChunk, fileLocation, partId, last); + logger.info("Chunk {} was successfully written to output stream {}.", partId, fileName); + uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); + if (!uploadTokenService.storeToken(uploadInfo, true)) { + // In this case data chunk was successfully appended to file (when LOCAL storage is used), but + // corresponding DB record contains incorrect data. This situation leads to data inconsistency. + // TODO: Check this behavior against cloud storage. (Google Cloud Storage). + logger.error( + "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); + return new ResponseEntity<>( + "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", + HttpStatus.INTERNAL_SERVER_ERROR); + } + } catch (Exception e) { + logger.error("Error writing data for file {}: {}", fileName, e.getMessage()); + logger.debug("Error writing data for file {}: {}", fileName, e.getCause()); + return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); + } + if (last) { + finalizeUpload(uploadInfo, fileLocation); + } + return new ResponseEntity<>(HttpStatus.CREATED); + } + + private String constructFileName(String fileName) { + String generatedPrefix = RandomStringUtils.randomAlphabetic(10); + return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); + } + + private String constructDownloadUrl(String fileKey) { + return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); + } + + private String retrieveFileName(UploadStore uploadInfo) { + String fileName; + if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // upload hasn't started yet + fileName = constructFileName(uploadInfo.getFileName()); + uploadInfo.setFileName(fileName); + } else { + fileName = uploadInfo.getFileName(); + } + return fileName; + } + + private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { + uploadInfo.setPartId(partId); + uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); + return uploadInfo; + + } + + private String finalizeUpload(UploadStore uploadInfo, String fileLocation) { + + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, uploadInfo.getFileName(), + fileLocation); + String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); + storageProvider.close(fileLocation); + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + logger.info("File sucessfully uploaded."); + return dowlnoadUrl; + } +} diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java b/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java new file mode 100644 index 0000000..6e325e2 --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest.validation; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import biz.nynja.content.upload.models.UploadStore; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class ChunkUploadValidator { + + private static final Logger logger = LoggerFactory.getLogger(ChunkUploadValidator.class); + + public Optional validateUploadRequest(UploadStore uploadInfo, byte[] data, String jobId, + String token, int partId, boolean last) { + + if (!jobId.equals(uploadInfo.getJobId().toString())) { + logger.error("Upload token {} does not apply to the requested job id: {}", token, jobId); + return Optional.of(new ValidationResult("Upload token does not apply to the requested upload URL.")); + } + + if (partId != uploadInfo.getPartId() + 1) { + logger.error( + "Chunk with Unexpected part id received for job id {} with associated token: {} . Expected: {}, actual: {}", + jobId, token, uploadInfo.getPartId() + 1, partId); + return Optional.of(new ValidationResult( + "You are trying to upload wrong data chunk. Last uploaded chunk " + uploadInfo.getPartId())); + } + if (data.length <= 0) { + logger.error("Recieved empty data chuck for jobId {} with associated token: {}", jobId, token); + return Optional.of(new ValidationResult("Missing chunk data.")); + } + + String sizeValidationResult = validateReachedFileSize(uploadInfo.getLastUploadedByte() + data.length, + uploadInfo.getFileSize(), last); + if (sizeValidationResult != null) { + logger.error("Incorrect file size reached for jobId {} with associated token: {}. {}", jobId, token, + sizeValidationResult); + return Optional.of(new ValidationResult(sizeValidationResult)); + } + + return Optional.empty(); + } + + private boolean matchesExpectedSize(int actual, int expected) { + return actual == expected; + } + + private boolean exceedsExpectedSize(int actual, int expected) { + return actual > expected; + } + + private String validateReachedFileSize(int actual, int expected, boolean lastChunk) { + if (exceedsExpectedSize(actual, expected)) { + logger.error("Size of uploaded file so far exceeds the expected file size."); + return "Size of uploaded file so far exceeds the expected file size."; + } + if (lastChunk && !matchesExpectedSize(actual, expected)) { + logger.error("Uploaded file size does not meet expectations - expected: {}, actual: {}.", expected, actual); + return "Uploaded file size does not meet expectations."; + } + return null; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java b/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java new file mode 100644 index 0000000..5074bd9 --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest.validation; + +/** + * @author Ralitsa Todorova + * + */ +public class ValidationResult { + + private String message; + + public ValidationResult(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + +} diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 4fc172e..a4cfd5b 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -36,7 +36,7 @@ media-types: file: upload: - url: http://localhost:${grpc.port} + url: http://localhost:${grpc.port}/file/upload/ download: url: http://localhost:${server.port}/file/download/ diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 63f2d69..29d8d6c 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -36,7 +36,7 @@ media-types: file: upload: - url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net}:${grpc.port} + url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net/file/upload} download: url: ${FILE_DOWNLOAD_URL:https://content.dev-eu.nynja.net/file/download/} -- GitLab From 89459dab779f59948fe15684b6383bb56c6d877d Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 11:32:02 +0200 Subject: [PATCH 12/70] NY-6474: Fixed wording Signed-off-by: Ralitsa Todorova --- .../nynja/content/file/storage/impl/LocalStorageProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 0aa9656..e333ee6 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -44,7 +44,7 @@ public class LocalStorageProvider implements StorageProvider { FileOutputStream output = new FileOutputStream(fileLocation, true); try { output.write(data); - logger.info("Bytes were successfully append to file {}.", fileLocation); + logger.info("Bytes were successfully appended to file {}.", fileLocation); } finally { output.close(); } -- GitLab From 44a92487cafd667a1e22f3747549eeb2103e00b7 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 17:52:27 +0200 Subject: [PATCH 13/70] NY-6474: Add getProviderType method for Storage Provider classes Signed-off-by: Ralitsa Todorova --- .../biz/nynja/content/file/storage/StorageProvider.java | 2 ++ .../content/file/storage/impl/GoogleStorageProvider.java | 8 ++++++++ .../content/file/storage/impl/LocalStorageProvider.java | 8 ++++++++ 3 files changed, 18 insertions(+) diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index cc4df34..c62c516 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -16,4 +16,6 @@ public interface StorageProvider { public void close(String fileLocation); public String getFileUrl(String filename); + + public Provider getProviderType(); } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 785df48..0d971e7 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -28,6 +28,7 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.InputStreamContent; import com.google.api.services.storage.StorageScopes; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageConfiguration; import biz.nynja.content.file.storage.StorageProvider; import io.netty.buffer.ByteBufInputStream; @@ -43,6 +44,8 @@ public class GoogleStorageProvider implements StorageProvider { private static final Logger logger = LoggerFactory.getLogger(GoogleStorageProvider.class); + private static final Provider providerType = Provider.GOOGLE; + private HttpRequestFactory requestFactory; private StorageConfiguration storageConfiguration; @@ -168,4 +171,9 @@ public class GoogleStorageProvider implements StorageProvider { .append(storageConfiguration.getGoogleBucketName()).append("/").append(filename); return builder.toString(); } + + @Override + public Provider getProviderType() { + return providerType; + } } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index e333ee6..8d4bcbb 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -12,6 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageConfiguration; import biz.nynja.content.file.storage.StorageProvider; @@ -25,6 +26,8 @@ public class LocalStorageProvider implements StorageProvider { private static final Logger logger = LoggerFactory.getLogger(LocalStorageProvider.class); + private static final Provider providerType = Provider.LOCAL; + private StorageConfiguration storageConfiguration; public LocalStorageProvider(StorageConfiguration storageConfiguration) { @@ -63,4 +66,9 @@ public class LocalStorageProvider implements StorageProvider { return constructFilePath(storageConfiguration.getLocalStorageLocation(), filename); } + @Override + public Provider getProviderType() { + return providerType; + } + } -- GitLab From 6789d134326ea4e8cc3efd072c34db492d5320b3 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 17:53:17 +0200 Subject: [PATCH 14/70] NY-6474: Set correct port in file uplaod URL Signed-off-by: Ralitsa Todorova --- src/main/resources/application-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index a4cfd5b..77123f4 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -36,7 +36,7 @@ media-types: file: upload: - url: http://localhost:${grpc.port}/file/upload/ + url: http://localhost:${server.port}/file/upload/ download: url: http://localhost:${server.port}/file/download/ -- GitLab From 6ab48d95b7e391b1267c81b7118f2a52f7d19f81 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 17:55:58 +0200 Subject: [PATCH 15/70] NY-6474: Improve file metadata storing Signed-off-by: Ralitsa Todorova --- .../nynja/content/file/metadata/FileMetadataService.java | 4 ++-- .../nynja/content/grpc/services/ContentServiceImpl.java | 3 ++- .../nynja/content/upload/rest/FileUploadController.java | 9 ++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index 981dff1..427f71f 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -30,10 +30,10 @@ public class FileMetadataService { this.fileMetadataRepository = fileMetadataRepository; } - public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl, Provider storageProvider) { + public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileUrl, Provider storageProvider) { FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setKey(UUID.randomUUID()); - fileMetadata.setFileName(fileName); + fileMetadata.setFileName(uploadInfo.getFileName()); fileMetadata.setFileSize(uploadInfo.getFileSize()); fileMetadata.setMediaType(uploadInfo.getMediaType()); fileMetadata.setUploadedFrom(uploadInfo.getUploadedFrom()); diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 629a518..8188c84 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -217,7 +217,8 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas Status.OUT_OF_RANGE.withDescription("Uploaded file size does not meet expectations."))); return; } - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, storageProvider.getFileUrl(fileName), provider); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + storageProvider.getFileUrl(fileName), storageProvider.getProviderType()); responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index 40894bf..bace40e 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -111,7 +111,7 @@ public class FileUploadController { return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); } if (last) { - finalizeUpload(uploadInfo, fileLocation); + finalizeUpload(uploadInfo, storageProvider); } return new ResponseEntity<>(HttpStatus.CREATED); } @@ -144,12 +144,11 @@ public class FileUploadController { } - private String finalizeUpload(UploadStore uploadInfo, String fileLocation) { + private String finalizeUpload(UploadStore uploadInfo, StorageProvider provider) { - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, uploadInfo.getFileName(), - fileLocation); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + provider.getFileUrl(uploadInfo.getFileName()), provider.getProviderType()); String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); - storageProvider.close(fileLocation); uploadTokenService.deleteToken(uploadInfo.getUploadToken()); logger.info("File sucessfully uploaded."); return dowlnoadUrl; -- GitLab From a16e226e2bbf3a6c0995f4c95cd3e7bcd47e71fa Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 19:20:54 +0200 Subject: [PATCH 16/70] NY-6475: Expect accessToken to be passed as Authorization header Signed-off-by: Ralitsa Todorova --- .../grpc/constants/ContentServiceConstants.java | 2 +- .../interceptors/ContentServiceInterceptor.java | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java b/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java index eef546e..0bb2dd6 100644 --- a/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java +++ b/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java @@ -14,7 +14,7 @@ import io.grpc.Metadata; */ public class ContentServiceConstants { - public static final Metadata.Key ACCESS_TOKEN_METADATA = Metadata.Key.of("accessToken", + public static final Metadata.Key ACCESS_TOKEN_METADATA = Metadata.Key.of("Authorization", ASCII_STRING_MARSHALLER); public static final Context.Key ACCOUNT_ID = Context.key("accountId"); diff --git a/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java b/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java index f67e858..c0f0bfb 100644 --- a/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java +++ b/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java @@ -22,6 +22,7 @@ import io.grpc.ServerCall; import io.grpc.ServerCall.Listener; import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; +import io.grpc.Status; /** * @author Ralitsa Todorova @@ -40,15 +41,26 @@ public class ContentServiceInterceptor implements ServerInterceptor { @Override public Listener interceptCall(ServerCall call, Metadata headers, ServerCallHandler next) { - String accessToken = headers.get(ContentServiceConstants.ACCESS_TOKEN_METADATA); + String accessToken = getAccessToken(headers.get(ContentServiceConstants.ACCESS_TOKEN_METADATA)); Context ctx = Context.current(); if (!StringUtils.isEmpty(accessToken)) { ctx = Context.current().withValue(ContentServiceConstants.ACCOUNT_ID, retrieveAccountId(accessToken)); - } + } else + call.close(Status.PERMISSION_DENIED, new Metadata()); return Contexts.interceptCall(ctx, call, headers, next); } + private String getAccessToken(String authHeader) { + // Authorization header is expected to be in format -> "Authorization" : "Bearer <>" + String accessToken = null; + try { + accessToken = authHeader.split(" ")[1]; + } catch (Exception e) { + logger.error("Unexpected Authorization Header format: {}", authHeader); + } + return accessToken; + } private String retrieveAccountId(String accessToken) { DecodedJWT decodedToken = JWT.decode(accessToken); String accountIdEncoded = decodedToken.getSubject(); -- GitLab From fea6fde6442fba5f455b02c883590b0ec262eb95 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Mon, 21 Jan 2019 18:05:59 +0200 Subject: [PATCH 17/70] NY-6474: Refactor FileUploadController - Move buisness logic in a separate service class - Introduce UploadResponse Signed-off-by: Ralitsa Todorova --- .../content/upload/models/UploadStatus.java | 2 +- .../upload/rest/FileUploadController.java | 112 ++++------------- .../upload/rest/FileUploadService.java | 113 ++++++++++++++++++ .../content/upload/rest/UploadResponse.java | 112 +++++++++++++++++ 4 files changed, 246 insertions(+), 93 deletions(-) create mode 100644 src/main/java/biz/nynja/content/upload/rest/FileUploadService.java create mode 100644 src/main/java/biz/nynja/content/upload/rest/UploadResponse.java diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java b/src/main/java/biz/nynja/content/upload/models/UploadStatus.java index d0380b5..ac9e5e9 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java +++ b/src/main/java/biz/nynja/content/upload/models/UploadStatus.java @@ -9,5 +9,5 @@ package biz.nynja.content.upload.models; */ public enum UploadStatus { - PENDING, IN_PROGRESS, FAILED; + PENDING, IN_PROGRESS, FAILED, COMPLETED; } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index bace40e..107f3e3 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -7,7 +7,6 @@ import java.util.Optional; import javax.servlet.http.HttpServletRequest; -import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -19,12 +18,6 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import biz.nynja.content.file.metadata.FileMetadataService; -import biz.nynja.content.file.metadata.dto.FileMetadata; -import biz.nynja.content.file.storage.StorageProvider; -import biz.nynja.content.file.storage.StorageProviderPool; -import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; -import biz.nynja.content.upload.models.UploadStatus; import biz.nynja.content.upload.models.UploadStore; import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; import biz.nynja.content.upload.rest.validation.ValidationResult; @@ -40,27 +33,22 @@ public class FileUploadController { private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); - private StorageProvider storageProvider; - private StorageProviderPool storageProviderPool; - private ContentServiceConfiguration contentServiceConfiguration; private UploadTokenService uploadTokenService; - private FileMetadataService fileMetadataService; private ChunkUploadValidator validator; + private FileUploadService fileUploadService; + + public FileUploadController(UploadTokenService uploadTokenService, ChunkUploadValidator validator, + FileUploadService fileUploadService) { - public FileUploadController(StorageProviderPool storageProviderPool, - ContentServiceConfiguration contentServiceConfiguration, UploadTokenService uploadTokenService, - FileMetadataService fileMetadataService, ChunkUploadValidator validator) { - this.storageProviderPool = storageProviderPool; - this.contentServiceConfiguration = contentServiceConfiguration; this.uploadTokenService = uploadTokenService; - this.fileMetadataService = fileMetadataService; this.validator = validator; + this.fileUploadService = fileUploadService; } @PutMapping("/{jobId}") - public ResponseEntity uploadFile(@PathVariable String jobId, @RequestParam("token") String uploadToken, - @RequestParam("partId") int partId, @RequestParam("last") boolean last, HttpServletRequest request, - @RequestBody byte[] dataChunk) { + public ResponseEntity uploadFile(@PathVariable String jobId, + @RequestParam("token") String uploadToken, @RequestParam("partId") int partId, + @RequestParam("last") boolean last, HttpServletRequest request, @RequestBody byte[] dataChunk) { logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); UploadStore uploadInfo; @@ -70,87 +58,27 @@ public class FileUploadController { } else { logger.error("You are not allowed to upload data. Provided token is either wrong or missing."); return new ResponseEntity<>( - "You are not allowed to upload data. Provided token is either wrong or missing.", + new UploadResponse( + "You are not allowed to upload data. Provided token is either wrong or missing."), HttpStatus.BAD_REQUEST); } - + Optional requestValidation = validator.validateUploadRequest(uploadInfo, dataChunk, jobId, - uploadToken,partId, last); + uploadToken, partId, last); if (requestValidation.isPresent()) { - return new ResponseEntity<>(requestValidation.get().getMessage(), HttpStatus.BAD_REQUEST); + return new ResponseEntity<>(new UploadResponse(requestValidation.get().getMessage(), uploadInfo.getStatus(), + uploadInfo.getPartId()), HttpStatus.BAD_REQUEST); } - String fileName = retrieveFileName(uploadInfo); - String fileLocation; - try { - if (storageProvider == null) { - storageProvider = storageProviderPool - .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); - } - - fileLocation = storageProvider.initialize(fileName); - logger.info("Created output stream for file {}", fileLocation); - - storageProvider.write(dataChunk, fileLocation, partId, last); - logger.info("Chunk {} was successfully written to output stream {}.", partId, fileName); - uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); - if (!uploadTokenService.storeToken(uploadInfo, true)) { - // In this case data chunk was successfully appended to file (when LOCAL storage is used), but - // corresponding DB record contains incorrect data. This situation leads to data inconsistency. - // TODO: Check this behavior against cloud storage. (Google Cloud Storage). - logger.error( - "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); - return new ResponseEntity<>( - "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", - HttpStatus.INTERNAL_SERVER_ERROR); - } + UploadResponse result = fileUploadService.writeDataChunk(uploadInfo, dataChunk, partId, last); + return new ResponseEntity<>(result, HttpStatus.CREATED); } catch (Exception e) { - logger.error("Error writing data for file {}: {}", fileName, e.getMessage()); - logger.debug("Error writing data for file {}: {}", fileName, e.getCause()); - return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); + logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); + logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); + return new ResponseEntity<>(new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), + uploadInfo.getLastUploadedByte()), HttpStatus.INTERNAL_SERVER_ERROR); } - if (last) { - finalizeUpload(uploadInfo, storageProvider); - } - return new ResponseEntity<>(HttpStatus.CREATED); - } - - private String constructFileName(String fileName) { - String generatedPrefix = RandomStringUtils.randomAlphabetic(10); - return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); - } - - private String constructDownloadUrl(String fileKey) { - return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); } - private String retrieveFileName(UploadStore uploadInfo) { - String fileName; - if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // upload hasn't started yet - fileName = constructFileName(uploadInfo.getFileName()); - uploadInfo.setFileName(fileName); - } else { - fileName = uploadInfo.getFileName(); - } - return fileName; - } - - private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { - uploadInfo.setPartId(partId); - uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); - uploadInfo.setStatus(UploadStatus.IN_PROGRESS); - return uploadInfo; - - } - - private String finalizeUpload(UploadStore uploadInfo, StorageProvider provider) { - - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, - provider.getFileUrl(uploadInfo.getFileName()), provider.getProviderType()); - String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); - uploadTokenService.deleteToken(uploadInfo.getUploadToken()); - logger.info("File sucessfully uploaded."); - return dowlnoadUrl; - } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java new file mode 100644 index 0000000..00fc32e --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest; + +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import biz.nynja.content.file.metadata.FileMetadataService; +import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.storage.StorageProviderPool; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; +import biz.nynja.content.upload.models.UploadStatus; +import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class FileUploadService { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); + + private StorageProvider storageProvider; + private UploadTokenService uploadTokenService; + private FileMetadataService fileMetadataService; + private ContentServiceConfiguration contentServiceConfiguration; + + @Autowired + public FileUploadService(StorageProviderPool storageProviderPool, UploadTokenService uploadTokenService, + FileMetadataService fileMetadataService, ContentServiceConfiguration contentServiceConfiguration) { + this.uploadTokenService = uploadTokenService; + this.fileMetadataService = fileMetadataService; + this.contentServiceConfiguration = contentServiceConfiguration; + this.storageProvider = storageProviderPool + .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); + } + + public UploadResponse writeDataChunk(UploadStore uploadInfo, byte[] dataChunk, int partId, boolean last) + throws Exception { + String fileName = retrieveFileName(uploadInfo); + String fileLocation; + + fileLocation = storageProvider.initialize(fileName); + logger.info("Created output stream for file {}", fileLocation); + + storageProvider.write(dataChunk, fileLocation, partId, last); + logger.info("Chunk {} was successfully written to output stream {}.", partId, fileName); + uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); + if (!uploadTokenService.storeToken(uploadInfo, true)) { + // In this case data chunk was successfully appended to file (when LOCAL storage is used), but + // corresponding DB record contains incorrect data. This situation leads to data inconsistency. + // TODO: Check this behavior against cloud storage. (Google Cloud Storage). + logger.error( + "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); + return new UploadResponse( + "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", + UploadStatus.FAILED, uploadInfo.getPartId()); + + } + if (last) { + finalizeUpload(uploadInfo); + return new UploadResponse("File uplaoad successfully completed.", UploadStatus.COMPLETED, + uploadInfo.getPartId()); + } + + return new UploadResponse("Data chunk uplaoad successfully completed.", UploadStatus.IN_PROGRESS, + uploadInfo.getPartId()); + } + + public String finalizeUpload(UploadStore uploadInfo) { + + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + storageProvider.getFileUrl(uploadInfo.getFileName()), storageProvider.getProviderType()); + String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + logger.info("File sucessfully uploaded."); + return dowlnoadUrl; + } + + private String retrieveFileName(UploadStore uploadInfo) { + String fileName; + if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // upload hasn't started yet + fileName = constructFileName(uploadInfo.getFileName()); + uploadInfo.setFileName(fileName); + } else { + fileName = uploadInfo.getFileName(); + } + return fileName; + } + + private String constructFileName(String fileName) { + String generatedPrefix = RandomStringUtils.randomAlphabetic(10); + return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); + } + + private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { + uploadInfo.setPartId(partId); + uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); + return uploadInfo; + } + + private String constructDownloadUrl(String fileKey) { + return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java b/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java new file mode 100644 index 0000000..0cb0766 --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java @@ -0,0 +1,112 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest; + +import biz.nynja.content.upload.models.UploadStatus; + +/** + * @author Ralitsa Todorova + * + */ +public class UploadResponse { + + private String message; + private UploadStatus status; + private int lastSuccessfulChunk; + private String downloadLink; + + public UploadResponse(String message) { + this.message = message; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk, String downloadLink) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + this.downloadLink = downloadLink; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public UploadStatus getStatus() { + return status; + } + + public void setStatus(UploadStatus status) { + this.status = status; + } + + public int getLastSuccessfulChunk() { + return lastSuccessfulChunk; + } + + public void setLastSuccessfulChunk(int lastSuccessfulChunk) { + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public String getDownloadLink() { + return downloadLink; + } + + public void setDownloadLink(String downloadLink) { + this.downloadLink = downloadLink; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((downloadLink == null) ? 0 : downloadLink.hashCode()); + result = prime * result + lastSuccessfulChunk; + result = prime * result + ((message == null) ? 0 : message.hashCode()); + result = prime * result + ((status == null) ? 0 : status.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadResponse other = (UploadResponse) obj; + if (downloadLink == null) { + if (other.downloadLink != null) + return false; + } else if (!downloadLink.equals(other.downloadLink)) + return false; + if (lastSuccessfulChunk != other.lastSuccessfulChunk) + return false; + if (message == null) { + if (other.message != null) + return false; + } else if (!message.equals(other.message)) + return false; + if (status != other.status) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadResponse [message=").append(message).append(", status=").append(status) + .append(", lastSuccessfulChunk=").append(lastSuccessfulChunk).append(", downloadLink=") + .append(downloadLink).append("]").toString(); + } + +} -- GitLab From a42211a5ea1479b3e4d22c4f253dbe9fc00da82f Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Tue, 22 Jan 2019 16:35:03 +0200 Subject: [PATCH 18/70] NY-6474: Extend upload info with additional parameters - Add timestamp for last uploaded chunk; - Add uploadLocation Signed-off-by: Ralitsa Todorova --- .../content/upload/models/UploadStore.java | 37 ++++++++++++++++++- .../CustomizedUploadStoreRepositoryImpl.java | 5 ++- .../upload/rest/FileUploadController.java | 2 +- .../upload/rest/FileUploadService.java | 36 +++++++++--------- 4 files changed, 58 insertions(+), 22 deletions(-) diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/upload/models/UploadStore.java index ec9c10a..66f4757 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/upload/models/UploadStore.java @@ -30,6 +30,8 @@ public class UploadStore { private int retries; private int partId; private int lastUploadedByte; + private String uploadLocation; + private Long lastUploadedChunkTimestamp; public UploadStore() { } @@ -50,6 +52,8 @@ public class UploadStore { this.retries = 0; this.partId = 0; this.lastUploadedByte = 0; + this.uploadLocation = ""; + this.lastUploadedChunkTimestamp = 0L; } public String getUploadToken() { @@ -164,6 +168,22 @@ public class UploadStore { this.lastUploadedByte = lastUploadedByte; } + public String getUploadLocation() { + return uploadLocation; + } + + public void setUploadLocation(String uploadLocation) { + this.uploadLocation = uploadLocation; + } + + public Long getLastUploadedChunkTimestamp() { + return lastUploadedChunkTimestamp; + } + + public void setLastUploadedChunkTimestamp(Long lastUploadedChunkTimestamp) { + this.lastUploadedChunkTimestamp = lastUploadedChunkTimestamp; + } + @Override public int hashCode() { final int prime = 31; @@ -174,11 +194,13 @@ public class UploadStore { result = prime * result + fileSize; result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); result = prime * result + lastUploadedByte; + result = prime * result + ((lastUploadedChunkTimestamp == null) ? 0 : lastUploadedChunkTimestamp.hashCode()); result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); result = prime * result + partId; result = prime * result + retries; result = prime * result + ((status == null) ? 0 : status.hashCode()); result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); + result = prime * result + ((uploadLocation == null) ? 0 : uploadLocation.hashCode()); result = prime * result + ((uploadToken == null) ? 0 : uploadToken.hashCode()); result = prime * result + ((uploadUrl == null) ? 0 : uploadUrl.hashCode()); result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); @@ -218,6 +240,11 @@ public class UploadStore { return false; if (lastUploadedByte != other.lastUploadedByte) return false; + if (lastUploadedChunkTimestamp == null) { + if (other.lastUploadedChunkTimestamp != null) + return false; + } else if (!lastUploadedChunkTimestamp.equals(other.lastUploadedChunkTimestamp)) + return false; if (mediaType == null) { if (other.mediaType != null) return false; @@ -234,6 +261,11 @@ public class UploadStore { return false; } else if (!tokenExpirationTime.equals(other.tokenExpirationTime)) return false; + if (uploadLocation == null) { + if (other.uploadLocation != null) + return false; + } else if (!uploadLocation.equals(other.uploadLocation)) + return false; if (uploadToken == null) { if (other.uploadToken != null) return false; @@ -259,8 +291,9 @@ public class UploadStore { .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") - .append(retries).append(", partId=").append(partId).append(", lastUploadedByte=") - .append(lastUploadedByte).append("]").toString(); + .append(retries).append(", partId=").append(partId) + .append(", lastUploadedByte=").append(lastUploadedByte) + .append("]").toString(); } } diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java b/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java index e82580e..29a19eb 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java +++ b/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java @@ -35,14 +35,15 @@ public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadS } private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { - String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte, lastuploadedchunktimestamp, uploadlocation) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + ttl + ";"; return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), uploadStore.getStatus().toString(), uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), uploadStore.getUploadUrl(), uploadStore.getRetries(), uploadStore.getPartId(), - uploadStore.getLastUploadedByte()); + uploadStore.getLastUploadedByte(), uploadStore.getLastUploadedChunkTimestamp(), + uploadStore.getUploadLocation()); } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index 107f3e3..9ddefbd 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -77,7 +77,7 @@ public class FileUploadController { logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); return new ResponseEntity<>(new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), - uploadInfo.getLastUploadedByte()), HttpStatus.INTERNAL_SERVER_ERROR); + uploadInfo.getPartId()), HttpStatus.INTERNAL_SERVER_ERROR); } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java index 00fc32e..2afc695 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java @@ -3,6 +3,8 @@ */ package biz.nynja.content.upload.rest; +import java.time.Instant; + import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,14 +46,14 @@ public class FileUploadService { public UploadResponse writeDataChunk(UploadStore uploadInfo, byte[] dataChunk, int partId, boolean last) throws Exception { - String fileName = retrieveFileName(uploadInfo); - String fileLocation; - fileLocation = storageProvider.initialize(fileName); - logger.info("Created output stream for file {}", fileLocation); + if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // Upload hasn't started yet + uploadInfo = initializeUpload(uploadInfo); + } + + storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last); - storageProvider.write(dataChunk, fileLocation, partId, last); - logger.info("Chunk {} was successfully written to output stream {}.", partId, fileName); + logger.info("Chunk {} was successfully written to output stream {}.", partId, uploadInfo.getFileName()); uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); if (!uploadTokenService.storeToken(uploadInfo, true)) { // In this case data chunk was successfully appended to file (when LOCAL storage is used), but @@ -74,6 +76,16 @@ public class FileUploadService { uploadInfo.getPartId()); } + private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { + String fileName = constructFileName(uploadInfo.getFileName()); + uploadInfo.setFileName(fileName); + + String uploadLocation = storageProvider.initialize(fileName); + uploadInfo.setUploadLocation(uploadLocation); + + return uploadInfo; + } + public String finalizeUpload(UploadStore uploadInfo) { FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, @@ -84,17 +96,6 @@ public class FileUploadService { return dowlnoadUrl; } - private String retrieveFileName(UploadStore uploadInfo) { - String fileName; - if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // upload hasn't started yet - fileName = constructFileName(uploadInfo.getFileName()); - uploadInfo.setFileName(fileName); - } else { - fileName = uploadInfo.getFileName(); - } - return fileName; - } - private String constructFileName(String fileName) { String generatedPrefix = RandomStringUtils.randomAlphabetic(10); return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); @@ -103,6 +104,7 @@ public class FileUploadService { private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { uploadInfo.setPartId(partId); uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); + uploadInfo.setLastUploadedChunkTimestamp(Instant.now().toEpochMilli()); uploadInfo.setStatus(UploadStatus.IN_PROGRESS); return uploadInfo; } -- GitLab From 173ecec30ec156c9ed7150eabc992b82204d5272 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Tue, 22 Jan 2019 16:43:41 +0200 Subject: [PATCH 19/70] NY-6474: Set correct header for access token in unit tests Signed-off-by: Ralitsa Todorova --- .../nynja/content/grpc/services/ContentServiceImplTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java index b5dd68f..631d9ab 100644 --- a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java +++ b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java @@ -83,8 +83,9 @@ public class ContentServiceImplTests extends GrpcServerTestBase { ContentServiceGrpc.ContentServiceBlockingStub contentServiceBlockingStub = ContentServiceGrpc .newBlockingStub(Optional.ofNullable(channel).orElse(inProcChannel)); Metadata header = new Metadata(); - Metadata.Key key = Metadata.Key.of("accessToken", Metadata.ASCII_STRING_MARSHALLER); - header.put(key, + Metadata.Key key = Metadata.Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER); + header.put(key, "Bearer " + + "eyJraWQiOiIyMDE4MDYwOCIsInR5cCI6IkpXVCIsImFsZyI6IkVTMjU2In0.eyJzdWIiOiJPVE5sWW1Jek5UQXRPVFZoT1MwME0ySmxMV0kzTkRjdFpEVmhPREV5TW1aalpqWTIiLCJhdWQiOiJNVEl6TVRJejphcHBDbGFzczoxMjMzMzMzIiwic2NvcGUiOiJhY2Nlc3MiLCJyb2xlcyI6W10sImlzcyI6Imh0dHBzOi8vYXV0aC5ueW5qYS5iaXovIiwiZXhwIjoxNTQyMDM4NDQ2LCJpYXQiOjE1NDIwMzQ4NDZ9._NB7HrnLeLNlHMk8bvxqrLn01TX97Y0bzylsaGiA6aEFKedKo8QciLePiLPyEEFHllpGk1bSRnqdmJjGQvuX7A"); this.contentServiceBlockingStub = MetadataUtils.attachHeaders(contentServiceBlockingStub, header); return; -- GitLab From a12368add55a47a08a657940f5ea3980c7d8381a Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 25 Jan 2019 14:24:11 +0200 Subject: [PATCH 20/70] NY-6478 Delete endpoint - cancel endpoint; - exception flow; - cancel function impl; Signed-off-by: abotev-intracol --- .../content/file/storage/StorageProvider.java | 4 ++- .../storage/impl/GoogleStorageProvider.java | 36 +++++++++---------- .../storage/impl/LocalStorageProvider.java | 8 ++++- .../grpc/services/ContentServiceImpl.java | 17 +++++++-- .../upload/rest/FileUploadController.java | 26 ++++++++++++-- .../upload/rest/FileUploadService.java | 17 +++++++-- 6 files changed, 79 insertions(+), 29 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index c62c516..6264d3f 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -3,6 +3,8 @@ */ package biz.nynja.content.file.storage; +import java.io.IOException; + /** * @author Angel.Botev * @@ -13,7 +15,7 @@ public interface StorageProvider { public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws Exception; - public void close(String fileLocation); + public void close(String fileLocation) throws IOException; public String getFileUrl(String filename); diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 0d971e7..6d0dc0a 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -56,7 +56,7 @@ public class GoogleStorageProvider implements StorageProvider { httpTransport = GoogleNetHttpTransport.newTrustedTransport(); // Build an account credential. GoogleCredential credential = GoogleCredential.getApplicationDefault(); - credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_READ_WRITE)); + credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)); requestFactory = httpTransport.createRequestFactory(credential); } catch (GeneralSecurityException | IOException e) { logger.error("Error with Google credentials: {}", e.getMessage()); @@ -82,7 +82,8 @@ public class GoogleStorageProvider implements StorageProvider { @Override public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws Exception { - logger.debug("Writing chunk: {} with length {} to: {}", chunkCount, data.length, fileLocation); + logger.debug("Writing partId: {} with length {} to: {}", chunkCount, data.length, fileLocation); + --chunkCount; // It's necessary because chunkCount starts from 1 not from 0 try (InputStream inputStream = new ByteBufInputStream(Unpooled.wrappedBuffer(data))) { int length = Math.min(Unpooled.wrappedBuffer(data).readableBytes(), storageConfiguration.getUploadChunkSize()); @@ -105,28 +106,23 @@ public class GoogleStorageProvider implements StorageProvider { } @Override - public void close(String fileLocation) { + public void close(String fileLocation) throws IOException { logger.debug("Cancel resumable Google upload for location: {}", fileLocation); String URI = fileLocation; GenericUrl url = new GenericUrl(URI); HttpRequest req; - try { - req = requestFactory.buildDeleteRequest(url); - HttpHeaders headers = new HttpHeaders(); - headers.setContentLength((long) 0); - req.setHeaders(headers); - logger.debug("Executing DELETE request to: {}", URI); - // Execute request - HttpResponse resp = req.execute(); - if (resp.getStatusCode() == 200) { - logger.debug("Successfull canceled Google upload for location: {}", fileLocation); - } else { - logger.error("Error canceling Google upload. Status code: {}, message: ", resp.getStatusCode(), - resp.getStatusMessage()); - } - } catch (IOException e) { - logger.error("Error canceling Google upload: {}", e.getMessage()); - logger.debug("Error canceling Google upload: {}", e.getCause()); + req = requestFactory.buildDeleteRequest(url); + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength((long) 0); + req.setHeaders(headers); + logger.debug("Executing DELETE request to: {}", URI); + // Execute request + HttpResponse resp = req.execute(); + if (resp.getStatusCode() == 200) { + logger.debug("Successfull canceled Google upload for location: {}", fileLocation); + } else { + logger.error("Error canceling Google upload. Status code: {}, message: ", resp.getStatusCode(), + resp.getStatusMessage()); } } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 8d4bcbb..9f4e24d 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -7,6 +7,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +56,11 @@ public class LocalStorageProvider implements StorageProvider { } @Override - public void close(String fileLocation) { + public void close(String fileLocation) throws IOException { + logger.debug("Cancel resumable upload for location: {}", fileLocation); + Files.delete(Paths.get(fileLocation)); + logger.debug("Successfull deleted file: {}", fileLocation); + logger.debug("Successfull canceled upload for location: {}", fileLocation); } private String constructFilePath(String contentStoreLocation, String fileName) { diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 8188c84..efc6978 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -5,6 +5,7 @@ package biz.nynja.content.grpc.services; import static biz.nynja.content.core.validation.Validators.util; +import java.io.IOException; import java.time.Instant; import java.util.Optional; import java.util.UUID; @@ -192,9 +193,9 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas return; } receivedBytesCount += data.length; + chunkCount++; storageProvider.write(data, fileLocation, chunkCount, (uploadInfo.getFileSize() == receivedBytesCount)); reachedSize += data.length; - chunkCount++; logger.info("Chunk {} was successfully written to output stream {}.", chunkCount, fileName); } catch (Exception e) { logger.error("Error writing data for file {}: {}", fileName, e.getMessage()); @@ -222,7 +223,12 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); - storageProvider.close(fileLocation); + try { + storageProvider.close(fileLocation); + } catch (IOException e) { + logger.error("Error closing upload job: {}", e.getMessage()); + logger.debug("Error closing upload job: {}", e.getCause()); + } uploadTokenService.deleteToken(uploadInfo.getUploadToken()); logger.info("File sucessfully uploaded."); } @@ -231,7 +237,12 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas uploadInfo.setRetries(uploadInfo.getRetries() + 1); uploadInfo.setStatus(UploadStatus.FAILED); uploadTokenService.storeToken(uploadInfo, false); - storageProvider.close(fileLocation); + try { + storageProvider.close(fileLocation); + } catch (IOException e) { + logger.error("Error closing upload job: {}", e.getMessage()); + logger.debug("Error closing upload job: {}", e.getCause()); + } } private String constructFileName(String fileName) { diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index 9ddefbd..657fe96 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -11,6 +11,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -76,9 +77,30 @@ public class FileUploadController { } catch (Exception e) { logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); - return new ResponseEntity<>(new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), - uploadInfo.getPartId()), HttpStatus.INTERNAL_SERVER_ERROR); + return new ResponseEntity<>( + new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), uploadInfo.getPartId()), + HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + @DeleteMapping("/{jobId}") + public ResponseEntity cancelUploading(@PathVariable String jobId, + @RequestParam("token") String uploadToken) { + + logger.info("Cancel request recived for jobId: {} and token: {}", jobId, uploadToken); + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (!uploadInfoResult.isPresent()) { + logger.error( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished."); + return new ResponseEntity<>( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished.", + HttpStatus.BAD_REQUEST); } + if(!fileUploadService.cancelUploadProcess(uploadInfoResult.get())) { + return new ResponseEntity<>("Error canceling upload job. File not found or not removed.", HttpStatus.INTERNAL_SERVER_ERROR); + } + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java index 2afc695..6ad963e 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java @@ -3,6 +3,7 @@ */ package biz.nynja.content.upload.rest; +import java.io.IOException; import java.time.Instant; import org.apache.commons.lang3.RandomStringUtils; @@ -68,11 +69,11 @@ public class FileUploadService { } if (last) { finalizeUpload(uploadInfo); - return new UploadResponse("File uplaoad successfully completed.", UploadStatus.COMPLETED, + return new UploadResponse("File upload successfully completed.", UploadStatus.COMPLETED, uploadInfo.getPartId()); } - return new UploadResponse("Data chunk uplaoad successfully completed.", UploadStatus.IN_PROGRESS, + return new UploadResponse("Data chunk upload successfully completed.", UploadStatus.IN_PROGRESS, uploadInfo.getPartId()); } @@ -112,4 +113,16 @@ public class FileUploadService { private String constructDownloadUrl(String fileKey) { return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); } + + public boolean cancelUploadProcess(UploadStore uploadInfo) { + try { + this.storageProvider.close(uploadInfo.getUploadLocation()); + } catch (IOException e) { + logger.error("Error canceling upload job: {}", e.getMessage()); + logger.debug("Error canceling upload job: {}", e.getCause()); + return false; + } + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + return true; + } } -- GitLab From 5c4c3a3736f11869fc12097a406f5e12769afe23 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Mon, 28 Jan 2019 12:03:09 +0200 Subject: [PATCH 21/70] NY-6479: REST upload unit tests Signed-off-by: Ralitsa Todorova --- .../upload/rest/FileUploadController.java | 4 +- .../rest/validation/ChunkUploadValidator.java | 6 + .../file/upload/FileUploadControllerTest.java | 199 ++++++++++++++++++ .../biz/nynja/content/file/upload/Util.java | 72 +++++++ 4 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java create mode 100644 src/test/java/biz/nynja/content/file/upload/Util.java diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index 9ddefbd..04350d6 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -5,8 +5,6 @@ package biz.nynja.content.upload.rest; import java.util.Optional; -import javax.servlet.http.HttpServletRequest; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -48,7 +46,7 @@ public class FileUploadController { @PutMapping("/{jobId}") public ResponseEntity uploadFile(@PathVariable String jobId, @RequestParam("token") String uploadToken, @RequestParam("partId") int partId, - @RequestParam("last") boolean last, HttpServletRequest request, @RequestBody byte[] dataChunk) { + @RequestParam("last") boolean last, @RequestBody byte[] dataChunk) { logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); UploadStore uploadInfo; diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java b/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java index 6e325e2..a044084 100644 --- a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java +++ b/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java @@ -68,6 +68,12 @@ public class ChunkUploadValidator { logger.error("Uploaded file size does not meet expectations - expected: {}, actual: {}.", expected, actual); return "Uploaded file size does not meet expectations."; } + if (!lastChunk && matchesExpectedSize(actual, expected)) { + logger.error( + "Uploaded file size so far reached expected overal file size, but this is not a final chunk - expected: {}, actual: {}.", + expected, actual); + return "Uploaded file size so far reached expected overal file size, but this is not a final chunk."; + } return null; } diff --git a/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java new file mode 100644 index 0000000..0963bde --- /dev/null +++ b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java @@ -0,0 +1,199 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload; + +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.Optional; + +import javax.annotation.Resource; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.web.context.WebApplicationContext; + +import biz.nynja.content.upload.models.UploadStatus; +import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.upload.rest.FileUploadController; +import biz.nynja.content.upload.rest.FileUploadService; +import biz.nynja.content.upload.rest.UploadResponse; +import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@RunWith(SpringRunner.class) +@WebMvcTest(FileUploadController.class) +@ContextConfiguration(classes = { FileUploadController.class, ChunkUploadValidator.class, Util.class }) +@ActiveProfiles("dev") +public class FileUploadControllerTest { + + @Resource + private WebApplicationContext wac; + + @MockBean + private FileUploadService fileUploadService; + + @MockBean + private UploadTokenService uploadTokenService; + + @Autowired + private MockMvc mockMvc; + + @Autowired + @Qualifier("uploadInfo") + private UploadStore uploadInfo; + + @Autowired + @Qualifier("uploadInfoInterm") + private UploadStore uploadInfoInterm; + + @Autowired + @Qualifier("uploadResponseFinalSuccess") + private UploadResponse uploadResponseFinalSuccess; + + @Autowired + @Qualifier("uploadResponseIntermediateChunkSuccess") + private UploadResponse uploadResponseIntermediateChunkSuccess; + + private final String UPLOAD_URL = "/file/upload/"; + + @Test + public void testUploadFinalChunkCreated() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + given(fileUploadService.writeDataChunk(uploadInfo, Util.DATA_CHUNK, 1, true)) + .willReturn(uploadResponseFinalSuccess); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, true)) + .content(Util.DATA_CHUNK)).andExpect(status().isCreated()); + } + + @Test + public void testUploadFinalChunkResponse() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + given(fileUploadService.writeDataChunk(uploadInfo, Util.DATA_CHUNK, 1, true)) + .willReturn(uploadResponseFinalSuccess); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, true)) + .content(Util.DATA_CHUNK)).andExpect(status().isCreated()) + .andExpect(jsonPath("$.message").value(uploadResponseFinalSuccess.getMessage())) + .andExpect(jsonPath("$.status").value(UploadStatus.COMPLETED.name())) + .andExpect(jsonPath("$.lastSuccessfulChunk").value(1)) + .andExpect(jsonPath("$.downloadLink").value(Util.DOWNLOAD_LINK)); + } + + @Test + public void testUploadIntermediateChunkOk() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfoInterm)); + given(fileUploadService.writeDataChunk(uploadInfoInterm, Util.DATA_CHUNK, 1, false)) + .willReturn(new UploadResponse("Success", UploadStatus.IN_PROGRESS, 1)); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isCreated()) + .andExpect(jsonPath("$.status").value(UploadStatus.IN_PROGRESS.name())) + .andExpect(jsonPath("$.lastSuccessfulChunk").value(1)).andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + @Test + public void testUploadIntermediateChunkReachedFileSize() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + given(fileUploadService.writeDataChunk(uploadInfo, Util.DATA_CHUNK, 1, false)) + .willReturn(new UploadResponse("Size not match.")); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + @Test + public void testUploadWrongDataChunk() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 2, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.status").value(UploadStatus.PENDING.name())) + .andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + @Test + public void testUploadEmptyBody() throws Exception { + + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false))) + .andExpect(status().isBadRequest()); + } + + @Test + public void testUploadInvalidJobId() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + mockMvc.perform(put(buildUploadRequest(Util.INVALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()); + } + + @Test + public void testUploadEmptyJobId() throws Exception { + + mockMvc.perform(put(buildUploadRequest("", Util.VALID_UPLOAD_TOKEN, 1, false)).content(Util.DATA_CHUNK)) + .andExpect(status().isNotFound()); + } + + @Test + public void testUploadEmptyToken() throws Exception { + + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, "", 1, false)).content(Util.DATA_CHUNK)) + .andExpect(status().isBadRequest()); + } + + @Test + public void testUploadEmptyLast() throws Exception { + + mockMvc.perform( + put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1)).content(Util.DATA_CHUNK)) + .andExpect(status().isBadRequest()); + } + + @Test + public void testUploadEmptyPartId() throws Exception { + + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()); + } + + @Test + public void testUploadInvalidUploadToken() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.INVALID_UPLOAD_TOKEN)).willReturn(Optional.empty()); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + private String buildUploadRequest(String jobId, String token, int partId, boolean last) { + return new StringBuilder(UPLOAD_URL).append(jobId).append("?token=").append(token).append("&partId=") + .append(partId).append("&last=").append(last).toString(); + } + + private String buildUploadRequest(String jobId, String token, boolean last) { + return new StringBuilder(UPLOAD_URL).append(jobId).append("?token=").append(token).append("&last=").append(last) + .toString(); + } + + private String buildUploadRequest(String jobId, String token, int partId) { + return new StringBuilder(UPLOAD_URL).append(jobId).append("?token=").append(token).append("&partId=") + .append(partId).toString(); + } +} diff --git a/src/test/java/biz/nynja/content/file/upload/Util.java b/src/test/java/biz/nynja/content/file/upload/Util.java new file mode 100644 index 0000000..9e9085a --- /dev/null +++ b/src/test/java/biz/nynja/content/file/upload/Util.java @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload; + +import java.util.Random; +import java.util.UUID; + +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; + +import biz.nynja.content.upload.models.UploadStatus; +import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.upload.rest.UploadResponse; + +/** + * @author Ralitsa Todorova + * + */ +@TestConfiguration +public class Util { + + public final static String VALID_UPLOAD_TOKEN = "qwertyuiopasdfghjklzxcvbnm"; + public final static String INVALID_UPLOAD_TOKEN = "qwertyuiopasdfghjklzxcvb"; + public final static String VALID_UPLOAD_JOBID = "3318f002-5fc1-4aff-bf70-9dbf728c29ca"; + public final static String INVALID_UPLOAD_JOBID = "3318f002-5fc1-4aff-bf70-9d"; + public final static int FILE_SIZE = 12345; + public final static String FILE_NAME = "somefilename.txt"; + public final static byte[] DATA_CHUNK = generateBytes(); + public final static String DOWNLOAD_LINK = "https://content.nynja.biz/file/download/kkk"; + + @Bean + public UploadStore uploadInfo() { + UploadStore uploadInfo = new UploadStore(); + uploadInfo.setStatus(UploadStatus.PENDING); + uploadInfo.setUploadToken(VALID_UPLOAD_TOKEN); + uploadInfo.setJobId(UUID.fromString(VALID_UPLOAD_JOBID)); + uploadInfo.setFileSize(FILE_SIZE); + uploadInfo.setFileName(FILE_NAME); + uploadInfo.setLastUploadedByte(0); + return uploadInfo; + } + + @Bean + public UploadStore uploadInfoInterm() { + UploadStore uploadInfo = new UploadStore(); + uploadInfo.setStatus(UploadStatus.PENDING); + uploadInfo.setUploadToken(VALID_UPLOAD_TOKEN); + uploadInfo.setJobId(UUID.fromString(VALID_UPLOAD_JOBID)); + uploadInfo.setFileSize(FILE_SIZE * 2); + uploadInfo.setFileName(FILE_NAME); + uploadInfo.setLastUploadedByte(0); + return uploadInfo; + } + @Bean + public UploadResponse uploadResponseIntermediateChunkSuccess() { + UploadResponse uploadResponse = new UploadResponse("Upload success.", UploadStatus.IN_PROGRESS, 1, null); + return uploadResponse; + } + + @Bean + public UploadResponse uploadResponseFinalSuccess() { + UploadResponse uploadResponse = new UploadResponse("Upload success.", UploadStatus.COMPLETED, 1, DOWNLOAD_LINK); + return uploadResponse; + } + + private static byte[] generateBytes() { + byte[] b = new byte[FILE_SIZE]; + new Random().nextBytes(b); + return b; + } +} -- GitLab From bc53bd8ec9308409ef32ec058f3488cbcca375a5 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Mon, 28 Jan 2019 13:53:20 +0200 Subject: [PATCH 22/70] NY-6478 Delete endpoint - fix comments; Signed-off-by: abotev-intracol --- .../nynja/content/file/storage/impl/LocalStorageProvider.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 9f4e24d..29c75aa 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -57,9 +57,8 @@ public class LocalStorageProvider implements StorageProvider { @Override public void close(String fileLocation) throws IOException { - logger.debug("Cancel resumable upload for location: {}", fileLocation); + logger.debug("Canceling resumable upload for location: {}", fileLocation); Files.delete(Paths.get(fileLocation)); - logger.debug("Successfull deleted file: {}", fileLocation); logger.debug("Successfull canceled upload for location: {}", fileLocation); } -- GitLab From 5118973853d7ecf64caf8a6abf7c3e10958ab986 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 30 Jan 2019 12:24:23 +0200 Subject: [PATCH 23/70] NY-6838 Move rest upload package to file->upload package - move package; - fix imports; - fix in ExceptionHandler; Signed-off-by: abotev-intracol --- ...a => FileControllersExceptionHandler.java} | 16 +- .../file/metadata/FileMetadataService.java | 2 +- .../upload/models/UploadStatus.java | 26 +- .../{ => file}/upload/models/UploadStore.java | 598 +++++++++--------- .../CustomizedUploadStoreRepository.java | 28 +- .../CustomizedUploadStoreRepositoryImpl.java | 98 +-- .../repositories/UploadStoreRepository.java | 40 +- .../upload/rest/FileUploadController.java | 208 +++--- .../upload/rest/FileUploadService.java | 256 ++++---- .../upload/rest/UploadResponse.java | 224 +++---- .../rest/validation/ChunkUploadValidator.java | 160 ++--- .../rest/validation/ValidationResult.java | 52 +- .../{ => file}/upload/token/UploadToken.java | 248 ++++---- .../token/UploadTokenResponseProvider.java | 68 +- .../upload/token/UploadTokenService.java | 202 +++--- .../upload/token/UploadTokenValidator.java | 254 ++++---- .../MediaTypesConfiguration.java | 2 +- .../UploadTokenConfiguration.java | 102 +-- .../grpc/interceptors/UploadInterceptor.java | 6 +- .../grpc/services/ContentServiceImpl.java | 14 +- .../download/FileDownloadControllerTest.java | 4 +- .../file/upload/FileUploadControllerTest.java | 14 +- .../biz/nynja/content/file/upload/Util.java | 6 +- .../services/ContentServiceImplTests.java | 6 +- .../upload/token/UploadTokenTests.java | 6 +- 25 files changed, 1322 insertions(+), 1318 deletions(-) rename src/main/java/biz/nynja/content/file/{download/FileControllerExceptionHandler.java => FileControllersExceptionHandler.java} (92%) rename src/main/java/biz/nynja/content/{ => file}/upload/models/UploadStatus.java (80%) rename src/main/java/biz/nynja/content/{ => file}/upload/models/UploadStore.java (96%) rename src/main/java/biz/nynja/content/{ => file}/upload/repositories/CustomizedUploadStoreRepository.java (81%) rename src/main/java/biz/nynja/content/{ => file}/upload/repositories/CustomizedUploadStoreRepositoryImpl.java (90%) rename src/main/java/biz/nynja/content/{ => file}/upload/repositories/UploadStoreRepository.java (80%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/FileUploadController.java (92%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/FileUploadService.java (94%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/UploadResponse.java (93%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/validation/ChunkUploadValidator.java (94%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/validation/ValidationResult.java (85%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/UploadToken.java (95%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/UploadTokenResponseProvider.java (94%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/UploadTokenService.java (91%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/UploadTokenValidator.java (94%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/configuration/MediaTypesConfiguration.java (96%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/configuration/UploadTokenConfiguration.java (92%) diff --git a/src/main/java/biz/nynja/content/file/download/FileControllerExceptionHandler.java b/src/main/java/biz/nynja/content/file/FileControllersExceptionHandler.java similarity index 92% rename from src/main/java/biz/nynja/content/file/download/FileControllerExceptionHandler.java rename to src/main/java/biz/nynja/content/file/FileControllersExceptionHandler.java index 9139882..bd1381c 100644 --- a/src/main/java/biz/nynja/content/file/download/FileControllerExceptionHandler.java +++ b/src/main/java/biz/nynja/content/file/FileControllersExceptionHandler.java @@ -1,7 +1,7 @@ /** * Copyright (C) 2018 Nynja Inc. All rights reserved. */ -package biz.nynja.content.file.download; +package biz.nynja.content.file; import java.io.FileNotFoundException; import java.util.MissingResourceException; @@ -25,9 +25,9 @@ import org.springframework.web.client.HttpServerErrorException; import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; @ControllerAdvice -public class FileControllerExceptionHandler { +public class FileControllersExceptionHandler { - private static final Logger logger = LoggerFactory.getLogger(FileControllerExceptionHandler.class); + private static final Logger logger = LoggerFactory.getLogger(FileControllersExceptionHandler.class); @ExceptionHandler({ IllegalArgumentException.class, MethodArgumentNotValidException.class, HttpMessageNotReadableException.class, ServletRequestBindingException.class }) @@ -58,14 +58,14 @@ public class FileControllerExceptionHandler { public static ResponseEntity handleNotFoundExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_FOUND, e), HttpStatus.NOT_FOUND); + return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_FOUND, e.getMessage()), HttpStatus.NOT_FOUND); } @ExceptionHandler({ HttpMediaTypeNotAcceptableException.class }) public ResponseEntity handleNotAcceptableExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_ACCEPTABLE, e), + return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_ACCEPTABLE, e.getMessage()), HttpStatus.NOT_ACCEPTABLE); } @@ -73,20 +73,20 @@ public class FileControllerExceptionHandler { public ResponseEntity handleUnsupportedMediaTypeExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.UNSUPPORTED_MEDIA_TYPE, e), + return new ResponseEntity(new ExceptionBody(HttpStatus.UNSUPPORTED_MEDIA_TYPE, e.getMessage()), HttpStatus.UNSUPPORTED_MEDIA_TYPE); } @ExceptionHandler({ MethodArgumentTypeMismatchException.class }) public ResponseEntity handleException(Exception e) { - return new ResponseEntity(new ExceptionBody(HttpStatus.BAD_REQUEST, e), HttpStatus.BAD_REQUEST); + return new ResponseEntity(new ExceptionBody(HttpStatus.BAD_REQUEST, e.getMessage()), HttpStatus.BAD_REQUEST); } @ExceptionHandler({ HttpServerErrorException.class, InternalError.class, Exception.class }) public ResponseEntity handleInternalServerExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.INTERNAL_SERVER_ERROR, e), + return new ResponseEntity(new ExceptionBody(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage()), HttpStatus.INTERNAL_SERVER_ERROR); } diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index 427f71f..7c885bd 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.metadata.repositories.FileMetadataReposiory; import biz.nynja.content.file.storage.Provider; -import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.file.upload.models.UploadStore; /** * @author Ralitsa Todorova diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java b/src/main/java/biz/nynja/content/file/upload/models/UploadStatus.java similarity index 80% rename from src/main/java/biz/nynja/content/upload/models/UploadStatus.java rename to src/main/java/biz/nynja/content/file/upload/models/UploadStatus.java index ac9e5e9..bcb595b 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java +++ b/src/main/java/biz/nynja/content/file/upload/models/UploadStatus.java @@ -1,13 +1,13 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.models; - -/** - * @author Ralitsa Todorova - * - */ -public enum UploadStatus { - - PENDING, IN_PROGRESS, FAILED, COMPLETED; -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.models; + +/** + * @author Ralitsa Todorova + * + */ +public enum UploadStatus { + + PENDING, IN_PROGRESS, FAILED, COMPLETED; +} diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java similarity index 96% rename from src/main/java/biz/nynja/content/upload/models/UploadStore.java rename to src/main/java/biz/nynja/content/file/upload/models/UploadStore.java index 66f4757..b902848 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java @@ -1,299 +1,299 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.models; - -import java.util.UUID; - -import org.springframework.data.cassandra.core.mapping.PrimaryKey; -import org.springframework.data.cassandra.core.mapping.Table; - -/** - * @author Ralitsa Todorova - * - */ -@Table -public class UploadStore { - - @PrimaryKey - private String uploadToken; - private Long tokenExpirationTime; - private String uploadUrl; - private UUID jobId; - private String deviceId; - private UUID accountId; - private String fileName; - private int fileSize; - private String mediaType; - private String uploadedFrom; - private UploadStatus status; - private int retries; - private int partId; - private int lastUploadedByte; - private String uploadLocation; - private Long lastUploadedChunkTimestamp; - - public UploadStore() { - } - - public UploadStore(String uploadToken, Long tokenExpirationTime, String uploadUrl, UUID jobId, String deviceId, - UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, UploadStatus status) { - this.uploadToken = uploadToken; - this.tokenExpirationTime = tokenExpirationTime; - this.uploadUrl = uploadUrl; - this.jobId = jobId; - this.deviceId = deviceId; - this.accountId = accountId; - this.fileName = fileName; - this.fileSize = fileSize; - this.mediaType = mediaType; - this.uploadedFrom = uploadedFrom; - this.status = status; - this.retries = 0; - this.partId = 0; - this.lastUploadedByte = 0; - this.uploadLocation = ""; - this.lastUploadedChunkTimestamp = 0L; - } - - public String getUploadToken() { - return uploadToken; - } - - public String getUploadUrl() { - return uploadUrl; - } - - public void setUploadUrl(String uploadUrl) { - this.uploadUrl = uploadUrl; - } - - public UUID getJobId() { - return jobId; - } - - public void setJobId(UUID jobId) { - this.jobId = jobId; - } - - public String getDeviceId() { - return deviceId; - } - - public void setDeviceId(String deviceId) { - this.deviceId = deviceId; - } - - public UUID getAccountId() { - return accountId; - } - - public void setAccountId(UUID accountId) { - this.accountId = accountId; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public int getFileSize() { - return fileSize; - } - - public void setFileSize(int fileSize) { - this.fileSize = fileSize; - } - - public String getMediaType() { - return mediaType; - } - - public void setMediaType(String mediaType) { - this.mediaType = mediaType; - } - - public String getUploadedFrom() { - return uploadedFrom; - } - - public void setUploadedFrom(String uploadedFrom) { - this.uploadedFrom = uploadedFrom; - } - - public UploadStatus getStatus() { - return status; - } - - public void setStatus(UploadStatus status) { - this.status = status; - } - - public void setUploadToken(String uploadToken) { - this.uploadToken = uploadToken; - } - - public Long getTokenExpirationTime() { - return tokenExpirationTime; - } - - public void setTokenExpirationTime(Long tokenExpirationTime) { - this.tokenExpirationTime = tokenExpirationTime; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - this.retries = retries; - } - - public int getPartId() { - return partId; - } - - public void setPartId(int partId) { - this.partId = partId; - } - - public int getLastUploadedByte() { - return lastUploadedByte; - } - - public void setLastUploadedByte(int lastUploadedByte) { - this.lastUploadedByte = lastUploadedByte; - } - - public String getUploadLocation() { - return uploadLocation; - } - - public void setUploadLocation(String uploadLocation) { - this.uploadLocation = uploadLocation; - } - - public Long getLastUploadedChunkTimestamp() { - return lastUploadedChunkTimestamp; - } - - public void setLastUploadedChunkTimestamp(Long lastUploadedChunkTimestamp) { - this.lastUploadedChunkTimestamp = lastUploadedChunkTimestamp; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((accountId == null) ? 0 : accountId.hashCode()); - result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); - result = prime * result + ((fileName == null) ? 0 : fileName.hashCode()); - result = prime * result + fileSize; - result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); - result = prime * result + lastUploadedByte; - result = prime * result + ((lastUploadedChunkTimestamp == null) ? 0 : lastUploadedChunkTimestamp.hashCode()); - result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); - result = prime * result + partId; - result = prime * result + retries; - result = prime * result + ((status == null) ? 0 : status.hashCode()); - result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); - result = prime * result + ((uploadLocation == null) ? 0 : uploadLocation.hashCode()); - result = prime * result + ((uploadToken == null) ? 0 : uploadToken.hashCode()); - result = prime * result + ((uploadUrl == null) ? 0 : uploadUrl.hashCode()); - result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - UploadStore other = (UploadStore) obj; - if (accountId == null) { - if (other.accountId != null) - return false; - } else if (!accountId.equals(other.accountId)) - return false; - if (deviceId == null) { - if (other.deviceId != null) - return false; - } else if (!deviceId.equals(other.deviceId)) - return false; - if (fileName == null) { - if (other.fileName != null) - return false; - } else if (!fileName.equals(other.fileName)) - return false; - if (fileSize != other.fileSize) - return false; - if (jobId == null) { - if (other.jobId != null) - return false; - } else if (!jobId.equals(other.jobId)) - return false; - if (lastUploadedByte != other.lastUploadedByte) - return false; - if (lastUploadedChunkTimestamp == null) { - if (other.lastUploadedChunkTimestamp != null) - return false; - } else if (!lastUploadedChunkTimestamp.equals(other.lastUploadedChunkTimestamp)) - return false; - if (mediaType == null) { - if (other.mediaType != null) - return false; - } else if (!mediaType.equals(other.mediaType)) - return false; - if (partId != other.partId) - return false; - if (retries != other.retries) - return false; - if (status != other.status) - return false; - if (tokenExpirationTime == null) { - if (other.tokenExpirationTime != null) - return false; - } else if (!tokenExpirationTime.equals(other.tokenExpirationTime)) - return false; - if (uploadLocation == null) { - if (other.uploadLocation != null) - return false; - } else if (!uploadLocation.equals(other.uploadLocation)) - return false; - if (uploadToken == null) { - if (other.uploadToken != null) - return false; - } else if (!uploadToken.equals(other.uploadToken)) - return false; - if (uploadUrl == null) { - if (other.uploadUrl != null) - return false; - } else if (!uploadUrl.equals(other.uploadUrl)) - return false; - if (uploadedFrom == null) { - if (other.uploadedFrom != null) - return false; - } else if (!uploadedFrom.equals(other.uploadedFrom)) - return false; - return true; - } - - @Override - public String toString() { - return new StringBuilder("UploadStore [uploadToken=").append(uploadToken).append(", tokenExpirationTime=") - .append(tokenExpirationTime).append(", uploadUrl=").append(uploadUrl).append(", jobId=").append(jobId) - .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") - .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) - .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") - .append(retries).append(", partId=").append(partId) - .append(", lastUploadedByte=").append(lastUploadedByte) - .append("]").toString(); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.models; + +import java.util.UUID; + +import org.springframework.data.cassandra.core.mapping.PrimaryKey; +import org.springframework.data.cassandra.core.mapping.Table; + +/** + * @author Ralitsa Todorova + * + */ +@Table +public class UploadStore { + + @PrimaryKey + private String uploadToken; + private Long tokenExpirationTime; + private String uploadUrl; + private UUID jobId; + private String deviceId; + private UUID accountId; + private String fileName; + private int fileSize; + private String mediaType; + private String uploadedFrom; + private UploadStatus status; + private int retries; + private int partId; + private int lastUploadedByte; + private String uploadLocation; + private Long lastUploadedChunkTimestamp; + + public UploadStore() { + } + + public UploadStore(String uploadToken, Long tokenExpirationTime, String uploadUrl, UUID jobId, String deviceId, + UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, UploadStatus status) { + this.uploadToken = uploadToken; + this.tokenExpirationTime = tokenExpirationTime; + this.uploadUrl = uploadUrl; + this.jobId = jobId; + this.deviceId = deviceId; + this.accountId = accountId; + this.fileName = fileName; + this.fileSize = fileSize; + this.mediaType = mediaType; + this.uploadedFrom = uploadedFrom; + this.status = status; + this.retries = 0; + this.partId = 0; + this.lastUploadedByte = 0; + this.uploadLocation = ""; + this.lastUploadedChunkTimestamp = 0L; + } + + public String getUploadToken() { + return uploadToken; + } + + public String getUploadUrl() { + return uploadUrl; + } + + public void setUploadUrl(String uploadUrl) { + this.uploadUrl = uploadUrl; + } + + public UUID getJobId() { + return jobId; + } + + public void setJobId(UUID jobId) { + this.jobId = jobId; + } + + public String getDeviceId() { + return deviceId; + } + + public void setDeviceId(String deviceId) { + this.deviceId = deviceId; + } + + public UUID getAccountId() { + return accountId; + } + + public void setAccountId(UUID accountId) { + this.accountId = accountId; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public int getFileSize() { + return fileSize; + } + + public void setFileSize(int fileSize) { + this.fileSize = fileSize; + } + + public String getMediaType() { + return mediaType; + } + + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + public String getUploadedFrom() { + return uploadedFrom; + } + + public void setUploadedFrom(String uploadedFrom) { + this.uploadedFrom = uploadedFrom; + } + + public UploadStatus getStatus() { + return status; + } + + public void setStatus(UploadStatus status) { + this.status = status; + } + + public void setUploadToken(String uploadToken) { + this.uploadToken = uploadToken; + } + + public Long getTokenExpirationTime() { + return tokenExpirationTime; + } + + public void setTokenExpirationTime(Long tokenExpirationTime) { + this.tokenExpirationTime = tokenExpirationTime; + } + + public int getRetries() { + return retries; + } + + public void setRetries(int retries) { + this.retries = retries; + } + + public int getPartId() { + return partId; + } + + public void setPartId(int partId) { + this.partId = partId; + } + + public int getLastUploadedByte() { + return lastUploadedByte; + } + + public void setLastUploadedByte(int lastUploadedByte) { + this.lastUploadedByte = lastUploadedByte; + } + + public String getUploadLocation() { + return uploadLocation; + } + + public void setUploadLocation(String uploadLocation) { + this.uploadLocation = uploadLocation; + } + + public Long getLastUploadedChunkTimestamp() { + return lastUploadedChunkTimestamp; + } + + public void setLastUploadedChunkTimestamp(Long lastUploadedChunkTimestamp) { + this.lastUploadedChunkTimestamp = lastUploadedChunkTimestamp; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((accountId == null) ? 0 : accountId.hashCode()); + result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); + result = prime * result + ((fileName == null) ? 0 : fileName.hashCode()); + result = prime * result + fileSize; + result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); + result = prime * result + lastUploadedByte; + result = prime * result + ((lastUploadedChunkTimestamp == null) ? 0 : lastUploadedChunkTimestamp.hashCode()); + result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); + result = prime * result + partId; + result = prime * result + retries; + result = prime * result + ((status == null) ? 0 : status.hashCode()); + result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); + result = prime * result + ((uploadLocation == null) ? 0 : uploadLocation.hashCode()); + result = prime * result + ((uploadToken == null) ? 0 : uploadToken.hashCode()); + result = prime * result + ((uploadUrl == null) ? 0 : uploadUrl.hashCode()); + result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadStore other = (UploadStore) obj; + if (accountId == null) { + if (other.accountId != null) + return false; + } else if (!accountId.equals(other.accountId)) + return false; + if (deviceId == null) { + if (other.deviceId != null) + return false; + } else if (!deviceId.equals(other.deviceId)) + return false; + if (fileName == null) { + if (other.fileName != null) + return false; + } else if (!fileName.equals(other.fileName)) + return false; + if (fileSize != other.fileSize) + return false; + if (jobId == null) { + if (other.jobId != null) + return false; + } else if (!jobId.equals(other.jobId)) + return false; + if (lastUploadedByte != other.lastUploadedByte) + return false; + if (lastUploadedChunkTimestamp == null) { + if (other.lastUploadedChunkTimestamp != null) + return false; + } else if (!lastUploadedChunkTimestamp.equals(other.lastUploadedChunkTimestamp)) + return false; + if (mediaType == null) { + if (other.mediaType != null) + return false; + } else if (!mediaType.equals(other.mediaType)) + return false; + if (partId != other.partId) + return false; + if (retries != other.retries) + return false; + if (status != other.status) + return false; + if (tokenExpirationTime == null) { + if (other.tokenExpirationTime != null) + return false; + } else if (!tokenExpirationTime.equals(other.tokenExpirationTime)) + return false; + if (uploadLocation == null) { + if (other.uploadLocation != null) + return false; + } else if (!uploadLocation.equals(other.uploadLocation)) + return false; + if (uploadToken == null) { + if (other.uploadToken != null) + return false; + } else if (!uploadToken.equals(other.uploadToken)) + return false; + if (uploadUrl == null) { + if (other.uploadUrl != null) + return false; + } else if (!uploadUrl.equals(other.uploadUrl)) + return false; + if (uploadedFrom == null) { + if (other.uploadedFrom != null) + return false; + } else if (!uploadedFrom.equals(other.uploadedFrom)) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadStore [uploadToken=").append(uploadToken).append(", tokenExpirationTime=") + .append(tokenExpirationTime).append(", uploadUrl=").append(uploadUrl).append(", jobId=").append(jobId) + .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") + .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) + .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") + .append(retries).append(", partId=").append(partId) + .append(", lastUploadedByte=").append(lastUploadedByte) + .append("]").toString(); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepository.java b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepository.java similarity index 81% rename from src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepository.java rename to src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepository.java index 793c38d..aae1f91 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepository.java +++ b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepository.java @@ -1,14 +1,14 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.repositories; - -/** - * @author Ralitsa Todorova - * @param - * - */ -public interface CustomizedUploadStoreRepository { - - S saveWithTtl(S uploadStore, int ttl); -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.repositories; + +/** + * @author Ralitsa Todorova + * @param + * + */ +public interface CustomizedUploadStoreRepository { + + S saveWithTtl(S uploadStore, int ttl); +} diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepositoryImpl.java similarity index 90% rename from src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java rename to src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepositoryImpl.java index 29a19eb..cd9c5b5 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java +++ b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepositoryImpl.java @@ -1,49 +1,49 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.repositories; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.cassandra.core.CassandraTemplate; - -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; - -/** - * @author Ralitsa Todorova - * - */ -public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadStoreRepository { - - private final CassandraTemplate cassandraTemplate; - private UploadTokenConfiguration uploadTokenConfiguration; - - @Autowired - public CustomizedUploadStoreRepositoryImpl(CassandraTemplate cassandraTemplate, - UploadTokenConfiguration uploadTokenConfiguration) { - this.cassandraTemplate = cassandraTemplate; - this.uploadTokenConfiguration = uploadTokenConfiguration; - } - - public S saveWithTtl(S uploadStore, int ttl) { - - if (!executeSaveWithTtl((UploadStore) uploadStore, uploadTokenConfiguration.getTokenTimeToLive())) { - throw new RuntimeException("Error inserting new upload info record."); - } else { - return uploadStore; - } - } - - private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { - String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte, lastuploadedchunktimestamp, uploadlocation) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " - + ttl + ";"; - return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), - uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), - uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), - uploadStore.getStatus().toString(), uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), - uploadStore.getUploadUrl(), uploadStore.getRetries(), uploadStore.getPartId(), - uploadStore.getLastUploadedByte(), uploadStore.getLastUploadedChunkTimestamp(), - uploadStore.getUploadLocation()); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.repositories; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.cassandra.core.CassandraTemplate; + +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; + +/** + * @author Ralitsa Todorova + * + */ +public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadStoreRepository { + + private final CassandraTemplate cassandraTemplate; + private UploadTokenConfiguration uploadTokenConfiguration; + + @Autowired + public CustomizedUploadStoreRepositoryImpl(CassandraTemplate cassandraTemplate, + UploadTokenConfiguration uploadTokenConfiguration) { + this.cassandraTemplate = cassandraTemplate; + this.uploadTokenConfiguration = uploadTokenConfiguration; + } + + public S saveWithTtl(S uploadStore, int ttl) { + + if (!executeSaveWithTtl((UploadStore) uploadStore, uploadTokenConfiguration.getTokenTimeToLive())) { + throw new RuntimeException("Error inserting new upload info record."); + } else { + return uploadStore; + } + } + + private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { + String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte, lastuploadedchunktimestamp, uploadlocation) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + + ttl + ";"; + return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), + uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), + uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), + uploadStore.getStatus().toString(), uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), + uploadStore.getUploadUrl(), uploadStore.getRetries(), uploadStore.getPartId(), + uploadStore.getLastUploadedByte(), uploadStore.getLastUploadedChunkTimestamp(), + uploadStore.getUploadLocation()); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/repositories/UploadStoreRepository.java b/src/main/java/biz/nynja/content/file/upload/repositories/UploadStoreRepository.java similarity index 80% rename from src/main/java/biz/nynja/content/upload/repositories/UploadStoreRepository.java rename to src/main/java/biz/nynja/content/file/upload/repositories/UploadStoreRepository.java index 9a78df4..bb5cef7 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/UploadStoreRepository.java +++ b/src/main/java/biz/nynja/content/file/upload/repositories/UploadStoreRepository.java @@ -1,20 +1,20 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.repositories; - -import org.springframework.data.cassandra.repository.CassandraRepository; -import org.springframework.stereotype.Repository; - -import biz.nynja.content.upload.models.UploadStore; - -/** - * @author Ralitsa Todorova - * - */ -@Repository -public interface UploadStoreRepository - extends CassandraRepository, CustomizedUploadStoreRepository { - - UploadStore findByUploadToken(String uploadToken); -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.repositories; + +import org.springframework.data.cassandra.repository.CassandraRepository; +import org.springframework.stereotype.Repository; + +import biz.nynja.content.file.upload.models.UploadStore; + +/** + * @author Ralitsa Todorova + * + */ +@Repository +public interface UploadStoreRepository + extends CassandraRepository, CustomizedUploadStoreRepository { + + UploadStore findByUploadToken(String uploadToken); +} diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java similarity index 92% rename from src/main/java/biz/nynja/content/upload/rest/FileUploadController.java rename to src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java index 0ba0e8b..759fd82 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java @@ -1,104 +1,104 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest; - -import java.util.Optional; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; -import biz.nynja.content.upload.rest.validation.ValidationResult; -import biz.nynja.content.upload.token.UploadTokenService; - -/** - * @author Ralitsa Todorova - * - */ -@RestController -@RequestMapping("/file/upload") -public class FileUploadController { - - private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); - - private UploadTokenService uploadTokenService; - private ChunkUploadValidator validator; - private FileUploadService fileUploadService; - - public FileUploadController(UploadTokenService uploadTokenService, ChunkUploadValidator validator, - FileUploadService fileUploadService) { - - this.uploadTokenService = uploadTokenService; - this.validator = validator; - this.fileUploadService = fileUploadService; - } - - @PutMapping("/{jobId}") - public ResponseEntity uploadFile(@PathVariable String jobId, - @RequestParam("token") String uploadToken, @RequestParam("partId") int partId, - @RequestParam("last") boolean last, @RequestBody byte[] dataChunk) { - - logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); - UploadStore uploadInfo; - Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); - if (uploadInfoResult.isPresent()) { - uploadInfo = uploadInfoResult.get(); - } else { - logger.error("You are not allowed to upload data. Provided token is either wrong or missing."); - return new ResponseEntity<>( - new UploadResponse( - "You are not allowed to upload data. Provided token is either wrong or missing."), - HttpStatus.BAD_REQUEST); - } - - Optional requestValidation = validator.validateUploadRequest(uploadInfo, dataChunk, jobId, - uploadToken, partId, last); - if (requestValidation.isPresent()) { - return new ResponseEntity<>(new UploadResponse(requestValidation.get().getMessage(), uploadInfo.getStatus(), - uploadInfo.getPartId()), HttpStatus.BAD_REQUEST); - } - - try { - UploadResponse result = fileUploadService.writeDataChunk(uploadInfo, dataChunk, partId, last); - return new ResponseEntity<>(result, HttpStatus.CREATED); - } catch (Exception e) { - logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); - logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); - return new ResponseEntity<>( - new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), uploadInfo.getPartId()), - HttpStatus.INTERNAL_SERVER_ERROR); - } - } - - @DeleteMapping("/{jobId}") - public ResponseEntity cancelUploading(@PathVariable String jobId, - @RequestParam("token") String uploadToken) { - - logger.info("Cancel request recived for jobId: {} and token: {}", jobId, uploadToken); - Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); - if (!uploadInfoResult.isPresent()) { - logger.error( - "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished."); - return new ResponseEntity<>( - "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished.", - HttpStatus.BAD_REQUEST); - } - if(!fileUploadService.cancelUploadProcess(uploadInfoResult.get())) { - return new ResponseEntity<>("Error canceling upload job. File not found or not removed.", HttpStatus.INTERNAL_SERVER_ERROR); - } - - return new ResponseEntity<>(HttpStatus.NO_CONTENT); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.file.upload.rest.validation.ValidationResult; +import biz.nynja.content.file.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@RestController +@RequestMapping("/file/upload") +public class FileUploadController { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); + + private UploadTokenService uploadTokenService; + private ChunkUploadValidator validator; + private FileUploadService fileUploadService; + + public FileUploadController(UploadTokenService uploadTokenService, ChunkUploadValidator validator, + FileUploadService fileUploadService) { + + this.uploadTokenService = uploadTokenService; + this.validator = validator; + this.fileUploadService = fileUploadService; + } + + @PutMapping("/{jobId}") + public ResponseEntity uploadFile(@PathVariable String jobId, + @RequestParam("token") String uploadToken, @RequestParam("partId") int partId, + @RequestParam("last") boolean last, @RequestBody byte[] dataChunk) { + + logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); + UploadStore uploadInfo; + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (uploadInfoResult.isPresent()) { + uploadInfo = uploadInfoResult.get(); + } else { + logger.error("You are not allowed to upload data. Provided token is either wrong or missing."); + return new ResponseEntity<>( + new UploadResponse( + "You are not allowed to upload data. Provided token is either wrong or missing."), + HttpStatus.BAD_REQUEST); + } + + Optional requestValidation = validator.validateUploadRequest(uploadInfo, dataChunk, jobId, + uploadToken, partId, last); + if (requestValidation.isPresent()) { + return new ResponseEntity<>(new UploadResponse(requestValidation.get().getMessage(), uploadInfo.getStatus(), + uploadInfo.getPartId()), HttpStatus.BAD_REQUEST); + } + + try { + UploadResponse result = fileUploadService.writeDataChunk(uploadInfo, dataChunk, partId, last); + return new ResponseEntity<>(result, HttpStatus.CREATED); + } catch (Exception e) { + logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); + logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); + return new ResponseEntity<>( + new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), uploadInfo.getPartId()), + HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + @DeleteMapping("/{jobId}") + public ResponseEntity cancelUploading(@PathVariable String jobId, + @RequestParam("token") String uploadToken) { + + logger.info("Cancel request recived for jobId: {} and token: {}", jobId, uploadToken); + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (!uploadInfoResult.isPresent()) { + logger.error( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished."); + return new ResponseEntity<>( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished.", + HttpStatus.BAD_REQUEST); + } + if(!fileUploadService.cancelUploadProcess(uploadInfoResult.get())) { + return new ResponseEntity<>("Error canceling upload job. File not found or not removed.", HttpStatus.INTERNAL_SERVER_ERROR); + } + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/rest/FileUploadService.java rename to src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index 6ad963e..1a235ef 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -1,128 +1,128 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest; - -import java.io.IOException; -import java.time.Instant; - -import org.apache.commons.lang3.RandomStringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import biz.nynja.content.file.metadata.FileMetadataService; -import biz.nynja.content.file.metadata.dto.FileMetadata; -import biz.nynja.content.file.storage.StorageProvider; -import biz.nynja.content.file.storage.StorageProviderPool; -import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; -import biz.nynja.content.upload.models.UploadStatus; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadTokenService; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class FileUploadService { - - private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); - - private StorageProvider storageProvider; - private UploadTokenService uploadTokenService; - private FileMetadataService fileMetadataService; - private ContentServiceConfiguration contentServiceConfiguration; - - @Autowired - public FileUploadService(StorageProviderPool storageProviderPool, UploadTokenService uploadTokenService, - FileMetadataService fileMetadataService, ContentServiceConfiguration contentServiceConfiguration) { - this.uploadTokenService = uploadTokenService; - this.fileMetadataService = fileMetadataService; - this.contentServiceConfiguration = contentServiceConfiguration; - this.storageProvider = storageProviderPool - .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); - } - - public UploadResponse writeDataChunk(UploadStore uploadInfo, byte[] dataChunk, int partId, boolean last) - throws Exception { - - if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // Upload hasn't started yet - uploadInfo = initializeUpload(uploadInfo); - } - - storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last); - - logger.info("Chunk {} was successfully written to output stream {}.", partId, uploadInfo.getFileName()); - uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); - if (!uploadTokenService.storeToken(uploadInfo, true)) { - // In this case data chunk was successfully appended to file (when LOCAL storage is used), but - // corresponding DB record contains incorrect data. This situation leads to data inconsistency. - // TODO: Check this behavior against cloud storage. (Google Cloud Storage). - logger.error( - "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); - return new UploadResponse( - "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", - UploadStatus.FAILED, uploadInfo.getPartId()); - - } - if (last) { - finalizeUpload(uploadInfo); - return new UploadResponse("File upload successfully completed.", UploadStatus.COMPLETED, - uploadInfo.getPartId()); - } - - return new UploadResponse("Data chunk upload successfully completed.", UploadStatus.IN_PROGRESS, - uploadInfo.getPartId()); - } - - private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { - String fileName = constructFileName(uploadInfo.getFileName()); - uploadInfo.setFileName(fileName); - - String uploadLocation = storageProvider.initialize(fileName); - uploadInfo.setUploadLocation(uploadLocation); - - return uploadInfo; - } - - public String finalizeUpload(UploadStore uploadInfo) { - - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, - storageProvider.getFileUrl(uploadInfo.getFileName()), storageProvider.getProviderType()); - String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); - uploadTokenService.deleteToken(uploadInfo.getUploadToken()); - logger.info("File sucessfully uploaded."); - return dowlnoadUrl; - } - - private String constructFileName(String fileName) { - String generatedPrefix = RandomStringUtils.randomAlphabetic(10); - return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); - } - - private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { - uploadInfo.setPartId(partId); - uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); - uploadInfo.setLastUploadedChunkTimestamp(Instant.now().toEpochMilli()); - uploadInfo.setStatus(UploadStatus.IN_PROGRESS); - return uploadInfo; - } - - private String constructDownloadUrl(String fileKey) { - return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); - } - - public boolean cancelUploadProcess(UploadStore uploadInfo) { - try { - this.storageProvider.close(uploadInfo.getUploadLocation()); - } catch (IOException e) { - logger.error("Error canceling upload job: {}", e.getMessage()); - logger.debug("Error canceling upload job: {}", e.getCause()); - return false; - } - uploadTokenService.deleteToken(uploadInfo.getUploadToken()); - return true; - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest; + +import java.io.IOException; +import java.time.Instant; + +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import biz.nynja.content.file.metadata.FileMetadataService; +import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.storage.StorageProviderPool; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class FileUploadService { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); + + private StorageProvider storageProvider; + private UploadTokenService uploadTokenService; + private FileMetadataService fileMetadataService; + private ContentServiceConfiguration contentServiceConfiguration; + + @Autowired + public FileUploadService(StorageProviderPool storageProviderPool, UploadTokenService uploadTokenService, + FileMetadataService fileMetadataService, ContentServiceConfiguration contentServiceConfiguration) { + this.uploadTokenService = uploadTokenService; + this.fileMetadataService = fileMetadataService; + this.contentServiceConfiguration = contentServiceConfiguration; + this.storageProvider = storageProviderPool + .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); + } + + public UploadResponse writeDataChunk(UploadStore uploadInfo, byte[] dataChunk, int partId, boolean last) + throws Exception { + + if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // Upload hasn't started yet + uploadInfo = initializeUpload(uploadInfo); + } + + storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last); + + logger.info("Chunk {} was successfully written to output stream {}.", partId, uploadInfo.getFileName()); + uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); + if (!uploadTokenService.storeToken(uploadInfo, true)) { + // In this case data chunk was successfully appended to file (when LOCAL storage is used), but + // corresponding DB record contains incorrect data. This situation leads to data inconsistency. + // TODO: Check this behavior against cloud storage. (Google Cloud Storage). + logger.error( + "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); + return new UploadResponse( + "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", + UploadStatus.FAILED, uploadInfo.getPartId()); + + } + if (last) { + finalizeUpload(uploadInfo); + return new UploadResponse("File upload successfully completed.", UploadStatus.COMPLETED, + uploadInfo.getPartId()); + } + + return new UploadResponse("Data chunk upload successfully completed.", UploadStatus.IN_PROGRESS, + uploadInfo.getPartId()); + } + + private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { + String fileName = constructFileName(uploadInfo.getFileName()); + uploadInfo.setFileName(fileName); + + String uploadLocation = storageProvider.initialize(fileName); + uploadInfo.setUploadLocation(uploadLocation); + + return uploadInfo; + } + + public String finalizeUpload(UploadStore uploadInfo) { + + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + storageProvider.getFileUrl(uploadInfo.getFileName()), storageProvider.getProviderType()); + String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + logger.info("File sucessfully uploaded."); + return dowlnoadUrl; + } + + private String constructFileName(String fileName) { + String generatedPrefix = RandomStringUtils.randomAlphabetic(10); + return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); + } + + private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { + uploadInfo.setPartId(partId); + uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); + uploadInfo.setLastUploadedChunkTimestamp(Instant.now().toEpochMilli()); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); + return uploadInfo; + } + + private String constructDownloadUrl(String fileKey) { + return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); + } + + public boolean cancelUploadProcess(UploadStore uploadInfo) { + try { + this.storageProvider.close(uploadInfo.getUploadLocation()); + } catch (IOException e) { + logger.error("Error canceling upload job: {}", e.getMessage()); + logger.debug("Error canceling upload job: {}", e.getCause()); + return false; + } + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + return true; + } +} diff --git a/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java b/src/main/java/biz/nynja/content/file/upload/rest/UploadResponse.java similarity index 93% rename from src/main/java/biz/nynja/content/upload/rest/UploadResponse.java rename to src/main/java/biz/nynja/content/file/upload/rest/UploadResponse.java index 0cb0766..0de3b12 100644 --- a/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/UploadResponse.java @@ -1,112 +1,112 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest; - -import biz.nynja.content.upload.models.UploadStatus; - -/** - * @author Ralitsa Todorova - * - */ -public class UploadResponse { - - private String message; - private UploadStatus status; - private int lastSuccessfulChunk; - private String downloadLink; - - public UploadResponse(String message) { - this.message = message; - } - - public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk) { - this.message = message; - this.status = status; - this.lastSuccessfulChunk = lastSuccessfulChunk; - } - - public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk, String downloadLink) { - this.message = message; - this.status = status; - this.lastSuccessfulChunk = lastSuccessfulChunk; - this.downloadLink = downloadLink; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public UploadStatus getStatus() { - return status; - } - - public void setStatus(UploadStatus status) { - this.status = status; - } - - public int getLastSuccessfulChunk() { - return lastSuccessfulChunk; - } - - public void setLastSuccessfulChunk(int lastSuccessfulChunk) { - this.lastSuccessfulChunk = lastSuccessfulChunk; - } - - public String getDownloadLink() { - return downloadLink; - } - - public void setDownloadLink(String downloadLink) { - this.downloadLink = downloadLink; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((downloadLink == null) ? 0 : downloadLink.hashCode()); - result = prime * result + lastSuccessfulChunk; - result = prime * result + ((message == null) ? 0 : message.hashCode()); - result = prime * result + ((status == null) ? 0 : status.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - UploadResponse other = (UploadResponse) obj; - if (downloadLink == null) { - if (other.downloadLink != null) - return false; - } else if (!downloadLink.equals(other.downloadLink)) - return false; - if (lastSuccessfulChunk != other.lastSuccessfulChunk) - return false; - if (message == null) { - if (other.message != null) - return false; - } else if (!message.equals(other.message)) - return false; - if (status != other.status) - return false; - return true; - } - - @Override - public String toString() { - return new StringBuilder("UploadResponse [message=").append(message).append(", status=").append(status) - .append(", lastSuccessfulChunk=").append(lastSuccessfulChunk).append(", downloadLink=") - .append(downloadLink).append("]").toString(); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest; + +import biz.nynja.content.file.upload.models.UploadStatus; + +/** + * @author Ralitsa Todorova + * + */ +public class UploadResponse { + + private String message; + private UploadStatus status; + private int lastSuccessfulChunk; + private String downloadLink; + + public UploadResponse(String message) { + this.message = message; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk, String downloadLink) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + this.downloadLink = downloadLink; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public UploadStatus getStatus() { + return status; + } + + public void setStatus(UploadStatus status) { + this.status = status; + } + + public int getLastSuccessfulChunk() { + return lastSuccessfulChunk; + } + + public void setLastSuccessfulChunk(int lastSuccessfulChunk) { + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public String getDownloadLink() { + return downloadLink; + } + + public void setDownloadLink(String downloadLink) { + this.downloadLink = downloadLink; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((downloadLink == null) ? 0 : downloadLink.hashCode()); + result = prime * result + lastSuccessfulChunk; + result = prime * result + ((message == null) ? 0 : message.hashCode()); + result = prime * result + ((status == null) ? 0 : status.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadResponse other = (UploadResponse) obj; + if (downloadLink == null) { + if (other.downloadLink != null) + return false; + } else if (!downloadLink.equals(other.downloadLink)) + return false; + if (lastSuccessfulChunk != other.lastSuccessfulChunk) + return false; + if (message == null) { + if (other.message != null) + return false; + } else if (!message.equals(other.message)) + return false; + if (status != other.status) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadResponse [message=").append(message).append(", status=").append(status) + .append(", lastSuccessfulChunk=").append(lastSuccessfulChunk).append(", downloadLink=") + .append(downloadLink).append("]").toString(); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java rename to src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java index a044084..22e0594 100644 --- a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java @@ -1,80 +1,80 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest.validation; - -import java.util.Optional; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import biz.nynja.content.upload.models.UploadStore; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class ChunkUploadValidator { - - private static final Logger logger = LoggerFactory.getLogger(ChunkUploadValidator.class); - - public Optional validateUploadRequest(UploadStore uploadInfo, byte[] data, String jobId, - String token, int partId, boolean last) { - - if (!jobId.equals(uploadInfo.getJobId().toString())) { - logger.error("Upload token {} does not apply to the requested job id: {}", token, jobId); - return Optional.of(new ValidationResult("Upload token does not apply to the requested upload URL.")); - } - - if (partId != uploadInfo.getPartId() + 1) { - logger.error( - "Chunk with Unexpected part id received for job id {} with associated token: {} . Expected: {}, actual: {}", - jobId, token, uploadInfo.getPartId() + 1, partId); - return Optional.of(new ValidationResult( - "You are trying to upload wrong data chunk. Last uploaded chunk " + uploadInfo.getPartId())); - } - if (data.length <= 0) { - logger.error("Recieved empty data chuck for jobId {} with associated token: {}", jobId, token); - return Optional.of(new ValidationResult("Missing chunk data.")); - } - - String sizeValidationResult = validateReachedFileSize(uploadInfo.getLastUploadedByte() + data.length, - uploadInfo.getFileSize(), last); - if (sizeValidationResult != null) { - logger.error("Incorrect file size reached for jobId {} with associated token: {}. {}", jobId, token, - sizeValidationResult); - return Optional.of(new ValidationResult(sizeValidationResult)); - } - - return Optional.empty(); - } - - private boolean matchesExpectedSize(int actual, int expected) { - return actual == expected; - } - - private boolean exceedsExpectedSize(int actual, int expected) { - return actual > expected; - } - - private String validateReachedFileSize(int actual, int expected, boolean lastChunk) { - if (exceedsExpectedSize(actual, expected)) { - logger.error("Size of uploaded file so far exceeds the expected file size."); - return "Size of uploaded file so far exceeds the expected file size."; - } - if (lastChunk && !matchesExpectedSize(actual, expected)) { - logger.error("Uploaded file size does not meet expectations - expected: {}, actual: {}.", expected, actual); - return "Uploaded file size does not meet expectations."; - } - if (!lastChunk && matchesExpectedSize(actual, expected)) { - logger.error( - "Uploaded file size so far reached expected overal file size, but this is not a final chunk - expected: {}, actual: {}.", - expected, actual); - return "Uploaded file size so far reached expected overal file size, but this is not a final chunk."; - } - return null; - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest.validation; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import biz.nynja.content.file.upload.models.UploadStore; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class ChunkUploadValidator { + + private static final Logger logger = LoggerFactory.getLogger(ChunkUploadValidator.class); + + public Optional validateUploadRequest(UploadStore uploadInfo, byte[] data, String jobId, + String token, int partId, boolean last) { + + if (!jobId.equals(uploadInfo.getJobId().toString())) { + logger.error("Upload token {} does not apply to the requested job id: {}", token, jobId); + return Optional.of(new ValidationResult("Upload token does not apply to the requested upload URL.")); + } + + if (partId != uploadInfo.getPartId() + 1) { + logger.error( + "Chunk with Unexpected part id received for job id {} with associated token: {} . Expected: {}, actual: {}", + jobId, token, uploadInfo.getPartId() + 1, partId); + return Optional.of(new ValidationResult( + "You are trying to upload wrong data chunk. Last uploaded chunk " + uploadInfo.getPartId())); + } + if (data.length <= 0) { + logger.error("Recieved empty data chuck for jobId {} with associated token: {}", jobId, token); + return Optional.of(new ValidationResult("Missing chunk data.")); + } + + String sizeValidationResult = validateReachedFileSize(uploadInfo.getLastUploadedByte() + data.length, + uploadInfo.getFileSize(), last); + if (sizeValidationResult != null) { + logger.error("Incorrect file size reached for jobId {} with associated token: {}. {}", jobId, token, + sizeValidationResult); + return Optional.of(new ValidationResult(sizeValidationResult)); + } + + return Optional.empty(); + } + + private boolean matchesExpectedSize(int actual, int expected) { + return actual == expected; + } + + private boolean exceedsExpectedSize(int actual, int expected) { + return actual > expected; + } + + private String validateReachedFileSize(int actual, int expected, boolean lastChunk) { + if (exceedsExpectedSize(actual, expected)) { + logger.error("Size of uploaded file so far exceeds the expected file size."); + return "Size of uploaded file so far exceeds the expected file size."; + } + if (lastChunk && !matchesExpectedSize(actual, expected)) { + logger.error("Uploaded file size does not meet expectations - expected: {}, actual: {}.", expected, actual); + return "Uploaded file size does not meet expectations."; + } + if (!lastChunk && matchesExpectedSize(actual, expected)) { + logger.error( + "Uploaded file size so far reached expected overal file size, but this is not a final chunk - expected: {}, actual: {}.", + expected, actual); + return "Uploaded file size so far reached expected overal file size, but this is not a final chunk."; + } + return null; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java b/src/main/java/biz/nynja/content/file/upload/rest/validation/ValidationResult.java similarity index 85% rename from src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java rename to src/main/java/biz/nynja/content/file/upload/rest/validation/ValidationResult.java index 5074bd9..288e0a4 100644 --- a/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/validation/ValidationResult.java @@ -1,26 +1,26 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest.validation; - -/** - * @author Ralitsa Todorova - * - */ -public class ValidationResult { - - private String message; - - public ValidationResult(String message) { - this.message = message; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest.validation; + +/** + * @author Ralitsa Todorova + * + */ +public class ValidationResult { + + private String message; + + public ValidationResult(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadToken.java b/src/main/java/biz/nynja/content/file/upload/token/UploadToken.java similarity index 95% rename from src/main/java/biz/nynja/content/upload/token/UploadToken.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadToken.java index 90ad1c0..ae63841 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadToken.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadToken.java @@ -1,124 +1,124 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -/** - * @author Ralitsa Todorova - * - */ -public class UploadToken { - - private String name; - private String mediaType; - private int size; - private String uploadedFrom; - private String deviceId; - - public UploadToken(String name, String mediaType, int size, String uploadedFrom, String deviceId) { - this.name = name; - this.mediaType = mediaType; - this.size = size; - this.uploadedFrom = uploadedFrom; - this.deviceId = deviceId; - } - - public String getDeviceId() { - return deviceId; - } - - public void setDeviceId(String deviceId) { - this.deviceId = deviceId; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getMediaType() { - return mediaType; - } - - public void setMediaType(String mediaType) { - this.mediaType = mediaType; - } - - public int getSize() { - return size; - } - - public void setSize(int size) { - this.size = size; - } - - public String getUploadedFrom() { - return uploadedFrom; - } - - public void setUploadedFrom(String uploadedFrom) { - this.uploadedFrom = uploadedFrom; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); - result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); - result = prime * result + ((name == null) ? 0 : name.hashCode()); - result = prime * result + size; - result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - UploadToken other = (UploadToken) obj; - if (deviceId == null) { - if (other.deviceId != null) - return false; - } else if (!deviceId.equals(other.deviceId)) - return false; - if (mediaType == null) { - if (other.mediaType != null) - return false; - } else if (!mediaType.equals(other.mediaType)) - return false; - if (name == null) { - if (other.name != null) - return false; - } else if (!name.equals(other.name)) - return false; - if (size != other.size) - return false; - if (uploadedFrom == null) { - if (other.uploadedFrom != null) - return false; - } else if (!uploadedFrom.equals(other.uploadedFrom)) - return false; - return true; - } - - @Override - public String toString() { - return new StringBuilder("UploadToken [name=").append(name).append(", mediaType=").append(mediaType) - .append(", size=").append(size).append(", uploadedFrom=").append(uploadedFrom).append(", deviceId=") - .append(deviceId).append("]").toString(); - } - - public String getTokenString() { - return new StringBuilder("n:").append(this.getName()).append(",mt:").append(this.getMediaType()).append(",s:") - .append(this.getSize()).append(",uf:").append(this.getUploadedFrom()).append(",did:") - .append(this.getDeviceId()).toString(); - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +/** + * @author Ralitsa Todorova + * + */ +public class UploadToken { + + private String name; + private String mediaType; + private int size; + private String uploadedFrom; + private String deviceId; + + public UploadToken(String name, String mediaType, int size, String uploadedFrom, String deviceId) { + this.name = name; + this.mediaType = mediaType; + this.size = size; + this.uploadedFrom = uploadedFrom; + this.deviceId = deviceId; + } + + public String getDeviceId() { + return deviceId; + } + + public void setDeviceId(String deviceId) { + this.deviceId = deviceId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getMediaType() { + return mediaType; + } + + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public String getUploadedFrom() { + return uploadedFrom; + } + + public void setUploadedFrom(String uploadedFrom) { + this.uploadedFrom = uploadedFrom; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); + result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + size; + result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadToken other = (UploadToken) obj; + if (deviceId == null) { + if (other.deviceId != null) + return false; + } else if (!deviceId.equals(other.deviceId)) + return false; + if (mediaType == null) { + if (other.mediaType != null) + return false; + } else if (!mediaType.equals(other.mediaType)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (size != other.size) + return false; + if (uploadedFrom == null) { + if (other.uploadedFrom != null) + return false; + } else if (!uploadedFrom.equals(other.uploadedFrom)) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadToken [name=").append(name).append(", mediaType=").append(mediaType) + .append(", size=").append(size).append(", uploadedFrom=").append(uploadedFrom).append(", deviceId=") + .append(deviceId).append("]").toString(); + } + + public String getTokenString() { + return new StringBuilder("n:").append(this.getName()).append(",mt:").append(this.getMediaType()).append(",s:") + .append(this.getSize()).append(",uf:").append(this.getUploadedFrom()).append(",did:") + .append(this.getDeviceId()).toString(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadTokenResponseProvider.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenResponseProvider.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/token/UploadTokenResponseProvider.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadTokenResponseProvider.java index 7cfb9b9..1582468 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadTokenResponseProvider.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenResponseProvider.java @@ -1,34 +1,34 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -import org.springframework.stereotype.Service; - -import biz.nynja.content.grpc.ErrorResponse; -import biz.nynja.content.grpc.ErrorResponse.Cause; -import biz.nynja.content.grpc.UploadToken; -import biz.nynja.content.grpc.UploadTokenResponse; -import io.grpc.stub.StreamObserver; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class UploadTokenResponseProvider { - - public void prepareUploadTokenResponse(StreamObserver responseObserver, String token, - String tokenUrl) { - responseObserver.onNext(UploadTokenResponse.newBuilder() - .setTokenDetails(UploadToken.newBuilder().setToken(token).setUploadUrl(tokenUrl)).build()); - responseObserver.onCompleted(); - } - - public void prepareErrorResponse(StreamObserver responseObserver, Cause error, - String message) { - responseObserver.onNext(UploadTokenResponse.newBuilder() - .setError(ErrorResponse.newBuilder().setCause(error).setMessage(message)).build()); - responseObserver.onCompleted(); - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +import org.springframework.stereotype.Service; + +import biz.nynja.content.grpc.ErrorResponse; +import biz.nynja.content.grpc.ErrorResponse.Cause; +import biz.nynja.content.grpc.UploadToken; +import biz.nynja.content.grpc.UploadTokenResponse; +import io.grpc.stub.StreamObserver; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class UploadTokenResponseProvider { + + public void prepareUploadTokenResponse(StreamObserver responseObserver, String token, + String tokenUrl) { + responseObserver.onNext(UploadTokenResponse.newBuilder() + .setTokenDetails(UploadToken.newBuilder().setToken(token).setUploadUrl(tokenUrl)).build()); + responseObserver.onCompleted(); + } + + public void prepareErrorResponse(StreamObserver responseObserver, Cause error, + String message) { + responseObserver.onNext(UploadTokenResponse.newBuilder() + .setError(ErrorResponse.newBuilder().setCause(error).setMessage(message)).build()); + responseObserver.onCompleted(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java similarity index 91% rename from src/main/java/biz/nynja/content/upload/token/UploadTokenService.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index f11abfd..fe306de 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -1,101 +1,101 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -import java.io.UnsupportedEncodingException; -import java.security.InvalidAlgorithmParameterException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.util.Optional; - -import javax.crypto.BadPaddingException; -import javax.crypto.IllegalBlockSizeException; -import javax.crypto.NoSuchPaddingException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import biz.nynja.content.security.EncryptDecryptService; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.repositories.UploadStoreRepository; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class UploadTokenService { - - private final Logger logger = LoggerFactory.getLogger(UploadTokenService.class); - - private EncryptDecryptService encryptDecryptService; - private UploadStoreRepository uploadStoreRepository; - private UploadTokenConfiguration uploadTokenConfiguration; - - public UploadTokenService(EncryptDecryptService encryptDecryptService, - UploadStoreRepository uploadStoreRepository, UploadTokenConfiguration uploadTokenConfiguration) { - this.encryptDecryptService = encryptDecryptService; - this.uploadStoreRepository = uploadStoreRepository; - this.uploadTokenConfiguration = uploadTokenConfiguration; - } - - public Optional prepareTokenResponse(UploadToken uploadToken) { - String result; - try { - result = encryptDecryptService.encrypt(uploadToken.getTokenString()); - logger.debug("Upload token successfully encrypted: {}", uploadToken.toString()); - } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException - | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException - | UnsupportedEncodingException e) { - logger.error("Error encrypting upload token. {}", e.getCause()); - logger.debug("Error encrypting upload token: {}", e.getMessage()); - return Optional.empty(); - } - - return Optional.of(result); - } - - public boolean storeToken(UploadStore uploadInfo, boolean permanent) { - try { - if (permanent) { - uploadStoreRepository.save(uploadInfo); - logger.info("Upload token successfully stored in DB."); - } else { - uploadStoreRepository.saveWithTtl(uploadInfo, uploadTokenConfiguration.getTokenTimeToLive()); - logger.info("Upload token successfully stored in DB for {} seconds.", - uploadTokenConfiguration.getTokenTimeToLive()); - } - logger.info("Upload token successfully stored in DB."); - return true; - } catch (Exception e) { - logger.error("Error storing record {} in DB: {}", uploadInfo.toString(), e.getMessage()); - logger.debug("Error storing record {} in DB: {}", uploadInfo.toString(), e.getCause()); - return false; - } - } - - public void deleteToken(String tokenId) { - try { - logger.debug("Deleting token {} from DB...", tokenId); - uploadStoreRepository.deleteById(tokenId); - logger.debug("Token {} successfully deleted from DB.", tokenId); - } catch (Exception e) { - logger.error("Error deleting token {} from DB: {}", tokenId, e.getMessage()); - logger.debug("Error deleting token {} from DB: {}", tokenId, e.getCause()); - } - - uploadStoreRepository.deleteById(tokenId); - } - - public Optional getUploadInfo(String uploadToken) { - - UploadStore storeInfo = uploadStoreRepository.findByUploadToken(uploadToken); - if (storeInfo != null) { - return Optional.of(storeInfo); - } - return Optional.empty(); - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +import java.io.UnsupportedEncodingException; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.Optional; + +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import biz.nynja.content.security.EncryptDecryptService; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.repositories.UploadStoreRepository; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class UploadTokenService { + + private final Logger logger = LoggerFactory.getLogger(UploadTokenService.class); + + private EncryptDecryptService encryptDecryptService; + private UploadStoreRepository uploadStoreRepository; + private UploadTokenConfiguration uploadTokenConfiguration; + + public UploadTokenService(EncryptDecryptService encryptDecryptService, + UploadStoreRepository uploadStoreRepository, UploadTokenConfiguration uploadTokenConfiguration) { + this.encryptDecryptService = encryptDecryptService; + this.uploadStoreRepository = uploadStoreRepository; + this.uploadTokenConfiguration = uploadTokenConfiguration; + } + + public Optional prepareTokenResponse(UploadToken uploadToken) { + String result; + try { + result = encryptDecryptService.encrypt(uploadToken.getTokenString()); + logger.debug("Upload token successfully encrypted: {}", uploadToken.toString()); + } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException + | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException + | UnsupportedEncodingException e) { + logger.error("Error encrypting upload token. {}", e.getCause()); + logger.debug("Error encrypting upload token: {}", e.getMessage()); + return Optional.empty(); + } + + return Optional.of(result); + } + + public boolean storeToken(UploadStore uploadInfo, boolean permanent) { + try { + if (permanent) { + uploadStoreRepository.save(uploadInfo); + logger.info("Upload token successfully stored in DB."); + } else { + uploadStoreRepository.saveWithTtl(uploadInfo, uploadTokenConfiguration.getTokenTimeToLive()); + logger.info("Upload token successfully stored in DB for {} seconds.", + uploadTokenConfiguration.getTokenTimeToLive()); + } + logger.info("Upload token successfully stored in DB."); + return true; + } catch (Exception e) { + logger.error("Error storing record {} in DB: {}", uploadInfo.toString(), e.getMessage()); + logger.debug("Error storing record {} in DB: {}", uploadInfo.toString(), e.getCause()); + return false; + } + } + + public void deleteToken(String tokenId) { + try { + logger.debug("Deleting token {} from DB...", tokenId); + uploadStoreRepository.deleteById(tokenId); + logger.debug("Token {} successfully deleted from DB.", tokenId); + } catch (Exception e) { + logger.error("Error deleting token {} from DB: {}", tokenId, e.getMessage()); + logger.debug("Error deleting token {} from DB: {}", tokenId, e.getCause()); + } + + uploadStoreRepository.deleteById(tokenId); + } + + public Optional getUploadInfo(String uploadToken) { + + UploadStore storeInfo = uploadStoreRepository.findByUploadToken(uploadToken); + if (storeInfo != null) { + return Optional.of(storeInfo); + } + return Optional.empty(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadTokenValidator.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenValidator.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/token/UploadTokenValidator.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadTokenValidator.java index a5a1f24..934838e 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadTokenValidator.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenValidator.java @@ -1,127 +1,127 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -import java.util.Collection; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; - -import com.google.common.io.Files; - -import biz.nynja.content.grpc.ErrorResponse.Cause; -import biz.nynja.content.core.validation.Validation; -import biz.nynja.content.core.validation.ValidationError; -import biz.nynja.content.grpc.UploadTokenRequest; -import biz.nynja.content.grpc.UploadTokenRequest.MediaType; -import biz.nynja.content.upload.token.configuration.MediaTypesConfiguration; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class UploadTokenValidator { - - private MediaTypesConfiguration mediaTypesConfiguration; - private UploadTokenConfiguration uploadTokenConfiguration; - - @Autowired - public UploadTokenValidator(MediaTypesConfiguration mediaTypesConfiguration, - UploadTokenConfiguration uploadTokenConfiguration) { - this.mediaTypesConfiguration = mediaTypesConfiguration; - this.uploadTokenConfiguration = uploadTokenConfiguration; - } - - public Validation validateGetTokenRequest(UploadTokenRequest request) { - Validation validation = new Validation(); - if (StringUtils.isEmpty(request.getName())) { - validation.addError(new ValidationError("Missing file name", Cause.MISSING_NAME)); - } - if (request.getSize() <= 0 || request.getSize() > uploadTokenConfiguration.getFileMaxSize()) { - validation.addError(new ValidationError("Incorrect file size", Cause.INCORRECT_SIZE)); - } - if (request.getMediaType().equals(MediaType.UNKNOWN_MEDIA_TYPE)) { - validation.addError(new ValidationError("Missing media type", Cause.MISSING_MEDIA_TYPE)); - } else { - String fileExtension = Files.getFileExtension(request.getName()); - Validation mediaTypeValidation = validateMediaType(request.getMediaType(), fileExtension); - if (mediaTypeValidation.hasErrors()) { - validation.addErrors(mediaTypeValidation.getErrors()); - } - } - if (StringUtils.isEmpty(request.getUploadedFrom())) { - validation.addError(new ValidationError("Missing uploadedFrom information", Cause.MISSING_UPLOADED_FROM)); - } - if (StringUtils.isEmpty(request.getDeviceId())) { - validation.addError(new ValidationError("Missing device id.", Cause.MISSING_DEVICE_ID)); - } - return validation; - } - - private Validation validateMediaType(MediaType requestMediaType, String fileExtension) { - Validation validation = new Validation(); - switch (requestMediaType) { - case AUDIO: - if (!mediaTypesConfiguration.getAudio().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case VIDEO: - if (!mediaTypesConfiguration.getVideo().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case COMPRESSED_FILES: - if (!mediaTypesConfiguration.getCompressed().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case DATA_FILES: - if (!mediaTypesConfiguration.getData().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case IMAGE: - if (!mediaTypesConfiguration.getImage().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case PAGE_LAYOUT: - if (!mediaTypesConfiguration.getPageLayout().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case SPREADSHEET: - if (!mediaTypesConfiguration.getSpreadsheet().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case TEXT: - if (!mediaTypesConfiguration.getText().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case OTHER: - if (Stream - .of(mediaTypesConfiguration.getAudio(), mediaTypesConfiguration.getData(), - mediaTypesConfiguration.getCompressed(), mediaTypesConfiguration.getImage(), - mediaTypesConfiguration.getPageLayout(), mediaTypesConfiguration.getSpreadsheet(), - mediaTypesConfiguration.getText(), mediaTypesConfiguration.getVideo()) - .flatMap(Collection::stream).collect(Collectors.toList()).contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - default: - break; - } - - return validation; - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import com.google.common.io.Files; + +import biz.nynja.content.grpc.ErrorResponse.Cause; +import biz.nynja.content.core.validation.Validation; +import biz.nynja.content.core.validation.ValidationError; +import biz.nynja.content.grpc.UploadTokenRequest; +import biz.nynja.content.grpc.UploadTokenRequest.MediaType; +import biz.nynja.content.file.upload.token.configuration.MediaTypesConfiguration; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class UploadTokenValidator { + + private MediaTypesConfiguration mediaTypesConfiguration; + private UploadTokenConfiguration uploadTokenConfiguration; + + @Autowired + public UploadTokenValidator(MediaTypesConfiguration mediaTypesConfiguration, + UploadTokenConfiguration uploadTokenConfiguration) { + this.mediaTypesConfiguration = mediaTypesConfiguration; + this.uploadTokenConfiguration = uploadTokenConfiguration; + } + + public Validation validateGetTokenRequest(UploadTokenRequest request) { + Validation validation = new Validation(); + if (StringUtils.isEmpty(request.getName())) { + validation.addError(new ValidationError("Missing file name", Cause.MISSING_NAME)); + } + if (request.getSize() <= 0 || request.getSize() > uploadTokenConfiguration.getFileMaxSize()) { + validation.addError(new ValidationError("Incorrect file size", Cause.INCORRECT_SIZE)); + } + if (request.getMediaType().equals(MediaType.UNKNOWN_MEDIA_TYPE)) { + validation.addError(new ValidationError("Missing media type", Cause.MISSING_MEDIA_TYPE)); + } else { + String fileExtension = Files.getFileExtension(request.getName()); + Validation mediaTypeValidation = validateMediaType(request.getMediaType(), fileExtension); + if (mediaTypeValidation.hasErrors()) { + validation.addErrors(mediaTypeValidation.getErrors()); + } + } + if (StringUtils.isEmpty(request.getUploadedFrom())) { + validation.addError(new ValidationError("Missing uploadedFrom information", Cause.MISSING_UPLOADED_FROM)); + } + if (StringUtils.isEmpty(request.getDeviceId())) { + validation.addError(new ValidationError("Missing device id.", Cause.MISSING_DEVICE_ID)); + } + return validation; + } + + private Validation validateMediaType(MediaType requestMediaType, String fileExtension) { + Validation validation = new Validation(); + switch (requestMediaType) { + case AUDIO: + if (!mediaTypesConfiguration.getAudio().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case VIDEO: + if (!mediaTypesConfiguration.getVideo().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case COMPRESSED_FILES: + if (!mediaTypesConfiguration.getCompressed().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case DATA_FILES: + if (!mediaTypesConfiguration.getData().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case IMAGE: + if (!mediaTypesConfiguration.getImage().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case PAGE_LAYOUT: + if (!mediaTypesConfiguration.getPageLayout().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case SPREADSHEET: + if (!mediaTypesConfiguration.getSpreadsheet().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case TEXT: + if (!mediaTypesConfiguration.getText().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case OTHER: + if (Stream + .of(mediaTypesConfiguration.getAudio(), mediaTypesConfiguration.getData(), + mediaTypesConfiguration.getCompressed(), mediaTypesConfiguration.getImage(), + mediaTypesConfiguration.getPageLayout(), mediaTypesConfiguration.getSpreadsheet(), + mediaTypesConfiguration.getText(), mediaTypesConfiguration.getVideo()) + .flatMap(Collection::stream).collect(Collectors.toList()).contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + default: + break; + } + + return validation; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/token/configuration/MediaTypesConfiguration.java b/src/main/java/biz/nynja/content/file/upload/token/configuration/MediaTypesConfiguration.java similarity index 96% rename from src/main/java/biz/nynja/content/upload/token/configuration/MediaTypesConfiguration.java rename to src/main/java/biz/nynja/content/file/upload/token/configuration/MediaTypesConfiguration.java index 9f0b431..e95a404 100644 --- a/src/main/java/biz/nynja/content/upload/token/configuration/MediaTypesConfiguration.java +++ b/src/main/java/biz/nynja/content/file/upload/token/configuration/MediaTypesConfiguration.java @@ -1,7 +1,7 @@ /** * Copyright (C) 2018 Nynja Inc. All rights reserved. */ -package biz.nynja.content.upload.token.configuration; +package biz.nynja.content.file.upload.token.configuration; import java.util.List; diff --git a/src/main/java/biz/nynja/content/upload/token/configuration/UploadTokenConfiguration.java b/src/main/java/biz/nynja/content/file/upload/token/configuration/UploadTokenConfiguration.java similarity index 92% rename from src/main/java/biz/nynja/content/upload/token/configuration/UploadTokenConfiguration.java rename to src/main/java/biz/nynja/content/file/upload/token/configuration/UploadTokenConfiguration.java index 27c1f81..54aabfc 100644 --- a/src/main/java/biz/nynja/content/upload/token/configuration/UploadTokenConfiguration.java +++ b/src/main/java/biz/nynja/content/file/upload/token/configuration/UploadTokenConfiguration.java @@ -1,51 +1,51 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token.configuration; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; - -/** - * @author Ralitsa Todorova - * - */ - -@Configuration -public class UploadTokenConfiguration { - - private final int tokenTimeToLive; - private final int fileMaxSize; - private final int maxRetries; - - @Autowired - public UploadTokenConfiguration(Environment env) { - this.tokenTimeToLive = parseProperty(env, "token.time_to_live"); - this.fileMaxSize = parseProperty(env, "token.max_file_size"); - this.maxRetries = parseProperty(env, "token.max_upload_retries"); - } - - private int parseProperty(Environment env, String property) throws InternalError { - int intProperty = 0; - try { - intProperty = Integer.parseInt(env.getRequiredProperty(property)); - } catch (NumberFormatException ex) { - throw new InternalError("Integer expected in " + property); - } - return intProperty; - - } - - public int getTokenTimeToLive() { - return tokenTimeToLive; - } - - public int getFileMaxSize() { - return fileMaxSize; - } - - public int getMaxRetries() { - return maxRetries; - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token.configuration; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +/** + * @author Ralitsa Todorova + * + */ + +@Configuration +public class UploadTokenConfiguration { + + private final int tokenTimeToLive; + private final int fileMaxSize; + private final int maxRetries; + + @Autowired + public UploadTokenConfiguration(Environment env) { + this.tokenTimeToLive = parseProperty(env, "token.time_to_live"); + this.fileMaxSize = parseProperty(env, "token.max_file_size"); + this.maxRetries = parseProperty(env, "token.max_upload_retries"); + } + + private int parseProperty(Environment env, String property) throws InternalError { + int intProperty = 0; + try { + intProperty = Integer.parseInt(env.getRequiredProperty(property)); + } catch (NumberFormatException ex) { + throw new InternalError("Integer expected in " + property); + } + return intProperty; + + } + + public int getTokenTimeToLive() { + return tokenTimeToLive; + } + + public int getFileMaxSize() { + return fileMaxSize; + } + + public int getMaxRetries() { + return maxRetries; + } +} diff --git a/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java b/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java index 9cfeda2..7c91613 100644 --- a/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java +++ b/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java @@ -11,9 +11,9 @@ import org.slf4j.LoggerFactory; import org.springframework.util.StringUtils; import biz.nynja.content.grpc.constants.ContentServiceConstants; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadTokenService; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; import io.grpc.Context; import io.grpc.Contexts; import io.grpc.Metadata; diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index efc6978..5713b2a 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -32,13 +32,13 @@ import biz.nynja.content.grpc.UploadTokenResponse; import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; import biz.nynja.content.grpc.constants.ContentServiceConstants; import biz.nynja.content.grpc.interceptors.UploadInterceptor; -import biz.nynja.content.upload.models.UploadStatus; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadToken; -import biz.nynja.content.upload.token.UploadTokenResponseProvider; -import biz.nynja.content.upload.token.UploadTokenService; -import biz.nynja.content.upload.token.UploadTokenValidator; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadToken; +import biz.nynja.content.file.upload.token.UploadTokenResponseProvider; +import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.token.UploadTokenValidator; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; import io.grpc.Status; import io.grpc.StatusException; import io.grpc.stub.StreamObserver; diff --git a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java index 3683322..ad3ce0c 100644 --- a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java +++ b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java @@ -27,13 +27,15 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; +import biz.nynja.content.file.FileControllersExceptionHandler; + /** * @author Angel.Botev * */ @RunWith(SpringRunner.class) @WebMvcTest(FileDownloadController.class) -@ContextConfiguration(classes = { FileDownloadController.class, FileControllerExceptionHandler.class }) +@ContextConfiguration(classes = { FileDownloadController.class, FileControllersExceptionHandler.class }) @ActiveProfiles("dev") public class FileDownloadControllerTest { diff --git a/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java index 0963bde..f28902d 100644 --- a/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java +++ b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java @@ -24,13 +24,13 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; -import biz.nynja.content.upload.models.UploadStatus; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.rest.FileUploadController; -import biz.nynja.content.upload.rest.FileUploadService; -import biz.nynja.content.upload.rest.UploadResponse; -import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; -import biz.nynja.content.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.FileUploadController; +import biz.nynja.content.file.upload.rest.FileUploadService; +import biz.nynja.content.file.upload.rest.UploadResponse; +import biz.nynja.content.file.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.file.upload.token.UploadTokenService; /** * @author Ralitsa Todorova diff --git a/src/test/java/biz/nynja/content/file/upload/Util.java b/src/test/java/biz/nynja/content/file/upload/Util.java index 9e9085a..47f42fb 100644 --- a/src/test/java/biz/nynja/content/file/upload/Util.java +++ b/src/test/java/biz/nynja/content/file/upload/Util.java @@ -9,9 +9,9 @@ import java.util.UUID; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; -import biz.nynja.content.upload.models.UploadStatus; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.rest.UploadResponse; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.UploadResponse; /** * @author Ralitsa Todorova diff --git a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java index 631d9ab..5eee2dc 100644 --- a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java +++ b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java @@ -26,9 +26,9 @@ import biz.nynja.content.grpc.ErrorResponse; import biz.nynja.content.grpc.UploadTokenRequest; import biz.nynja.content.grpc.UploadTokenResponse; import biz.nynja.content.grpc.services.util.ContentServiceUtil; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadToken; -import biz.nynja.content.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadToken; +import biz.nynja.content.file.upload.token.UploadTokenService; import io.grpc.Metadata; import io.grpc.stub.MetadataUtils; diff --git a/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java b/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java index 07ec4cc..1a6f12d 100644 --- a/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java +++ b/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java @@ -27,8 +27,10 @@ import org.springframework.test.context.junit4.SpringRunner; import biz.nynja.content.Util; import biz.nynja.content.security.EncryptDecryptConfiguration; import biz.nynja.content.security.EncryptDecryptService; -import biz.nynja.content.upload.repositories.UploadStoreRepository; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.file.upload.repositories.UploadStoreRepository; +import biz.nynja.content.file.upload.token.UploadToken; +import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; /** * @author Ralitsa Todorova -- GitLab From 9aacb28dc016a0f62b8f48e4319d9b71ac2e8347 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 29 Jan 2019 18:10:18 +0200 Subject: [PATCH 24/70] NY-6837 Remove old upload jobs - add restcontroller; - add delete old jobs function; - add getOldJobs function; Signed-off-by: abotev-intracol --- .../biz/nynja/content/file/JobScheduler.java | 42 ++++++++++++++++++ .../file/upload/token/UploadTokenService.java | 44 +++++++++++++++++-- .../ContentServiceConfiguration.java | 17 +++++++ src/main/resources/application-dev.yml | 3 ++ src/main/resources/application-production.yml | 2 + 5 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 src/main/java/biz/nynja/content/file/JobScheduler.java diff --git a/src/main/java/biz/nynja/content/file/JobScheduler.java b/src/main/java/biz/nynja/content/file/JobScheduler.java new file mode 100644 index 0000000..f2e8fea --- /dev/null +++ b/src/main/java/biz/nynja/content/file/JobScheduler.java @@ -0,0 +1,42 @@ +package biz.nynja.content.file; + +import java.util.List; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.RestController; + +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.FileUploadController; +import biz.nynja.content.file.upload.token.UploadTokenService; + +@RestController +public class JobScheduler { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); + + private UploadTokenService uploadTokenService; + + public JobScheduler(UploadTokenService uploadTokenService) { + this.uploadTokenService = uploadTokenService; + } + + @DeleteMapping("/jobs/old") + public ResponseEntity removeOldJobs() { + logger.info("Remove not finished (old) jobs "); + + Optional> oldJobs = uploadTokenService.getOldUploadJobs(); + if (oldJobs.isPresent()) { + logger.info("Founded old jobs! Countinue with delete process", oldJobs); + uploadTokenService.deleteJobs(oldJobs.get()); + return new ResponseEntity("Old jobs successfully deleted!", HttpStatus.NO_CONTENT); + } else { + logger.info("Old jobs didn't found!"); + return new ResponseEntity("Old jobs didn't found!", HttpStatus.NO_CONTENT); + } + } +} diff --git a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index fe306de..c651136 100644 --- a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -7,6 +7,10 @@ import java.io.UnsupportedEncodingException; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import javax.crypto.BadPaddingException; @@ -17,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; import biz.nynja.content.security.EncryptDecryptService; import biz.nynja.content.file.upload.models.UploadStore; import biz.nynja.content.file.upload.repositories.UploadStoreRepository; @@ -34,12 +39,15 @@ public class UploadTokenService { private EncryptDecryptService encryptDecryptService; private UploadStoreRepository uploadStoreRepository; private UploadTokenConfiguration uploadTokenConfiguration; + private ContentServiceConfiguration contentServiceConfiguration; - public UploadTokenService(EncryptDecryptService encryptDecryptService, - UploadStoreRepository uploadStoreRepository, UploadTokenConfiguration uploadTokenConfiguration) { + public UploadTokenService(EncryptDecryptService encryptDecryptService, UploadStoreRepository uploadStoreRepository, + UploadTokenConfiguration uploadTokenConfiguration, + ContentServiceConfiguration contentServiceConfiguration) { this.encryptDecryptService = encryptDecryptService; this.uploadStoreRepository = uploadStoreRepository; this.uploadTokenConfiguration = uploadTokenConfiguration; + this.contentServiceConfiguration = contentServiceConfiguration; } public Optional prepareTokenResponse(UploadToken uploadToken) { @@ -86,10 +94,20 @@ public class UploadTokenService { logger.error("Error deleting token {} from DB: {}", tokenId, e.getMessage()); logger.debug("Error deleting token {} from DB: {}", tokenId, e.getCause()); } + } - uploadStoreRepository.deleteById(tokenId); + public void deleteJobs(List jobs) { + try { + logger.debug("Deleting jobs {} from DB...", jobs); + uploadStoreRepository.deleteAll(jobs); + logger.debug("Jobs {} successfully deleted from DB.", jobs); + } catch (Exception e) { + logger.error("Error deleting jobs {} from DB: {}", jobs, e.getMessage()); + logger.debug("Error deleting jobs {} from DB: {}", jobs, e.getCause()); + } } + public Optional getUploadInfo(String uploadToken) { UploadStore storeInfo = uploadStoreRepository.findByUploadToken(uploadToken); @@ -98,4 +116,24 @@ public class UploadTokenService { } return Optional.empty(); } + + public Optional> getOldUploadJobs() { + + List jobs = uploadStoreRepository.findAll(); + if (jobs == null) { + return Optional.empty(); + } + + List oldJobs = new ArrayList<>(); + int jobTTL = contentServiceConfiguration.getJobTTL(); + Instant now = Instant.now(); + for (UploadStore job : jobs) { + Instant last = Instant.ofEpochMilli(job.getLastUploadedChunkTimestamp()); + long hours = ChronoUnit.HOURS.between(now, last); + if (Math.toIntExact(hours) > jobTTL) { + oldJobs.add(job); + } + } + return Optional.of(oldJobs); + } } diff --git a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java index e520288..e37e53f 100644 --- a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java +++ b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java @@ -19,12 +19,14 @@ public class ContentServiceConfiguration { private final String uploadUrl; private final String downloadUrl; private final Provider storageProvider; + private final int jobTTL; @Autowired public ContentServiceConfiguration(Environment env) { this.uploadUrl = env.getRequiredProperty("file.upload.url"); this.downloadUrl = env.getRequiredProperty("file.download.url"); this.storageProvider = getProvidersProperty("storage.provider", env); + this.jobTTL = parseProperty(env, "file.upload.job.ttl"); } private Provider getProvidersProperty(String key, Environment env) { @@ -32,6 +34,17 @@ public class ContentServiceConfiguration { return Provider.valueOf(provider); } + private int parseProperty(Environment env, String property) throws InternalError { + int intProperty = 0; + try { + intProperty = Integer.parseInt(env.getRequiredProperty(property)); + } catch (NumberFormatException ex) { + throw new InternalError("Integer expected in " + property); + } + return intProperty; + + } + public String getUploadUrl() { return uploadUrl; } @@ -43,4 +56,8 @@ public class ContentServiceConfiguration { public Provider getStorageProvider() { return storageProvider; } + + public int getJobTTL() { + return jobTTL; + } } diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 77123f4..b1f98b2 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -37,6 +37,8 @@ media-types: file: upload: url: http://localhost:${server.port}/file/upload/ + job: + ttl: 24 # measured in hours. download: url: http://localhost:${server.port}/file/download/ @@ -49,6 +51,7 @@ storage: bucket: content-service-dev upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + # To enable colors in Eclipse: # spring.output.ansi.enabled=ALWAYS and in eclipse # Help -> Install New Software... and click "Add..." to add the following URL: diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 29d8d6c..47bd9a5 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -37,6 +37,8 @@ media-types: file: upload: url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net/file/upload} + job: + ttl: 24 # measured in hours. download: url: ${FILE_DOWNLOAD_URL:https://content.dev-eu.nynja.net/file/download/} -- GitLab From 2b69bc3416762de6c725ae3240b294dbfa93914f Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Thu, 31 Jan 2019 11:59:08 +0200 Subject: [PATCH 25/70] NY-6837 Remove old upload jobs - rebase after package refactoring; Signed-off-by: abotev-intracol --- .../nynja/content/db/configuration/CassandraConfig.java | 2 +- .../{JobScheduler.java => upload/rest/JobController.java} | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) rename src/main/java/biz/nynja/content/file/{JobScheduler.java => upload/rest/JobController.java} (87%) diff --git a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java index fd051ff..516887a 100644 --- a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java +++ b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java @@ -64,7 +64,7 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Override public String[] getEntityBasePackages() { - return new String[] { "biz.nynja.content.upload.models", "biz.nynja.content.file.metadata.dto" }; + return new String[] { "biz.nynja.content.file.upload.models", "biz.nynja.content.file.metadata.dto" }; } public String getConfiguredKeyspaceName() { diff --git a/src/main/java/biz/nynja/content/file/JobScheduler.java b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java similarity index 87% rename from src/main/java/biz/nynja/content/file/JobScheduler.java rename to src/main/java/biz/nynja/content/file/upload/rest/JobController.java index f2e8fea..f815a16 100644 --- a/src/main/java/biz/nynja/content/file/JobScheduler.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java @@ -1,4 +1,4 @@ -package biz.nynja.content.file; +package biz.nynja.content.file.upload.rest; import java.util.List; import java.util.Optional; @@ -11,17 +11,16 @@ import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.RestController; import biz.nynja.content.file.upload.models.UploadStore; -import biz.nynja.content.file.upload.rest.FileUploadController; import biz.nynja.content.file.upload.token.UploadTokenService; @RestController -public class JobScheduler { +public class JobController { private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); private UploadTokenService uploadTokenService; - public JobScheduler(UploadTokenService uploadTokenService) { + public JobController(UploadTokenService uploadTokenService) { this.uploadTokenService = uploadTokenService; } -- GitLab From d0aade5459bd955687f2e2c6c1a283ea1fffd594 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 5 Feb 2019 17:04:23 +0200 Subject: [PATCH 26/70] Stop JWT authentication Signed-off-by: abotev-intracol --- .../templates/authentication-policy.yaml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/charts/content-service/templates/authentication-policy.yaml b/charts/content-service/templates/authentication-policy.yaml index df399c2..895af58 100644 --- a/charts/content-service/templates/authentication-policy.yaml +++ b/charts/content-service/templates/authentication-policy.yaml @@ -1,19 +1,19 @@ -apiVersion: "authentication.istio.io/v1alpha1" -kind: "Policy" -metadata: - name: {{ template "service.name" . }} - labels: - app: {{ template "service.name" . }} - chart: {{ template "service.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - targets: - - name: {{ template "service.name" . }} - origins: - - jwt: - issuer: https://auth.nynja.biz/ - jwksUri: http://auth-service.auth.svc.cluster.local:8008/keys/public - audiences: - - dGVzdEluc3RhbmNl:NynjaApp:NynjaOrg - principalBinding: USE_ORIGIN +#apiVersion: "authentication.istio.io/v1alpha1" +#kind: "Policy" +#metadata: +# name: {{ template "service.name" . }} +# labels: +# app: {{ template "service.name" . }} +# chart: {{ template "service.chart" . }} +# release: {{ .Release.Name }} +# heritage: {{ .Release.Service }} +#spec: +# targets: +# - name: {{ template "service.name" . }} +# origins: +# - jwt: +# issuer: https://auth.nynja.biz/ +# jwksUri: http://auth-service.auth.svc.cluster.local:8008/keys/public +# audiences: +# - dGVzdEluc3RhbmNl:NynjaApp:NynjaOrg +# principalBinding: USE_ORIGIN -- GitLab From acbc8d091d9293fd3418913e3a4cb66220b898fe Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 6 Feb 2019 16:12:57 +0200 Subject: [PATCH 27/70] NY-6843 Encode filename in DB - encode filename before write in DB; - decode filename when work with it; - fix file download to set right content type and filename; Signed-off-by: abotev-intracol --- .../file/download/FileDownloadController.java | 18 ++++++++++++++---- .../file/upload/rest/FileUploadService.java | 8 +++++--- .../file/upload/token/UploadTokenService.java | 14 ++++++++++++++ .../grpc/services/ContentServiceImpl.java | 5 +++-- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java index 5933218..0ea9039 100644 --- a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java +++ b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java @@ -12,6 +12,7 @@ import java.util.UUID; import javax.servlet.http.HttpServletRequest; +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -26,6 +27,8 @@ import org.springframework.web.bind.annotation.RestController; import com.nimbusds.jose.util.Base64URL; +import biz.nynja.content.file.upload.token.UploadTokenService; + /** * @author Angel.Botev * @@ -38,10 +41,12 @@ public class FileDownloadController { private static final Logger logger = LoggerFactory.getLogger(FileDownloadController.class); private FileDownloadService fileDownloadService; + private UploadTokenService uploadTokenService; @Autowired - public FileDownloadController(FileDownloadService fileDownloadService) { + public FileDownloadController(FileDownloadService fileDownloadService, UploadTokenService uploadTokenService) { this.fileDownloadService = fileDownloadService; + this.uploadTokenService = uploadTokenService; } @RequestMapping(method = RequestMethod.GET, value = "/{fileKey}") @@ -62,9 +67,11 @@ public class FileDownloadController { } // Try to determine file's content type String contentType = null; + String fileName = null; try { - contentType = request.getServletContext().getMimeType(resource.getFile().getAbsolutePath()); - } catch (IOException ex) { + fileName = uploadTokenService.decodeFileName(resource.getFile().getName()); + contentType = request.getServletContext().getMimeType(fileName); + } catch (Exception ex) { logger.info("Could not determine file type."); } @@ -72,10 +79,13 @@ public class FileDownloadController { if (contentType == null) { contentType = "application/octet-stream"; } + if (fileName == null) { + fileName = RandomStringUtils.randomAlphabetic(15); + } return ResponseEntity.ok().contentType(MediaType.parseMediaType(contentType)) .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + Base64URL.encode(resource.getFilename()) + "\"") + "attachment; filename=\"" + Base64URL.encode(fileName) + "\"") .body(resource); } } diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index 1a235ef..f665d10 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -78,10 +78,12 @@ public class FileUploadService { } private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { - String fileName = constructFileName(uploadInfo.getFileName()); - uploadInfo.setFileName(fileName); + String fileName = uploadTokenService.decodeFileName(uploadInfo.getFileName()); + String constructedFilename = constructFileName(fileName); + String encodedFileName = uploadTokenService.encodeFileName(constructedFilename); + uploadInfo.setFileName(encodedFileName); - String uploadLocation = storageProvider.initialize(fileName); + String uploadLocation = storageProvider.initialize(encodedFileName); uploadInfo.setUploadLocation(uploadLocation); return uploadInfo; diff --git a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index c651136..f878c42 100644 --- a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -10,6 +10,7 @@ import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Optional; @@ -136,4 +137,17 @@ public class UploadTokenService { } return Optional.of(oldJobs); } + + public String encodeFileName(String fileName) { + String encodedFilename = Base64.getEncoder().encodeToString(fileName.getBytes()); + logger.debug("Filename: {} was encoded to: {}", fileName, encodedFilename); + return encodedFilename; + } + + public String decodeFileName(String encodedFilename) { + byte[] decodedFilenameInBytes = Base64.getDecoder().decode(encodedFilename); + String decodedFilename = new String(decodedFilenameInBytes); + logger.debug("Encoded filename: {} was decoded to: {}", encodedFilename, encodedFilename); + return decodedFilename; + } } diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 5713b2a..b301e5b 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -87,6 +87,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas request.getMediaType(), request.getSize(), request.getUploadedFrom(), request.getDeviceId()); String accountId = ContentServiceConstants.ACCOUNT_ID.get(); + String fileName = uploadTokenService.encodeFileName(request.getName()); if (StringUtils.isEmpty(accountId) || !util.isValidUuid(accountId)) { uploadTokenResponseProvider.prepareErrorResponse(responseObserver, Cause.INTERNAL_SERVER_ERROR, @@ -102,7 +103,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas validationResult.getErrorMessage()); return; } - UploadToken token = new UploadToken(request.getName(), request.getMediaType().name(), request.getSize(), + UploadToken token = new UploadToken(fileName, request.getMediaType().name(), request.getSize(), request.getUploadedFrom(), request.getDeviceId()); Optional tokenResponse = uploadTokenService.prepareTokenResponse(token); if (!tokenResponse.isPresent()) { @@ -115,7 +116,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas UploadStore uploadInfo = new UploadStore(tokenResponse.get(), Instant.now().toEpochMilli() + uploadTokenConfiguration.getTokenTimeToLive(), contentSreviceConfiguration.getUploadUrl() + jobId, jobId, request.getDeviceId(), - UUID.fromString(accountId), request.getName(), request.getSize(), request.getMediaType().name(), + UUID.fromString(accountId), fileName, request.getSize(), request.getMediaType().name(), request.getUploadedFrom(), UploadStatus.PENDING); if (!uploadTokenService.storeToken(uploadInfo, false)) { -- GitLab From 79d616457498a3c1bb00af063b03727d92ab3e1a Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 6 Feb 2019 18:08:59 +0200 Subject: [PATCH 28/70] NY-6843 Encode filename in DB - fix comments Signed-off-by: abotev-intracol --- .../biz/nynja/content/file/upload/token/UploadTokenService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index f878c42..df9487f 100644 --- a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -147,7 +147,7 @@ public class UploadTokenService { public String decodeFileName(String encodedFilename) { byte[] decodedFilenameInBytes = Base64.getDecoder().decode(encodedFilename); String decodedFilename = new String(decodedFilenameInBytes); - logger.debug("Encoded filename: {} was decoded to: {}", encodedFilename, encodedFilename); + logger.debug("Encoded filename: {} was decoded to: {}", encodedFilename, decodedFilename); return decodedFilename; } } -- GitLab From 3b5a7cb760174e6691fa6fa8c2a5d4dfc25fc7bf Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Fri, 8 Feb 2019 14:11:24 +0000 Subject: [PATCH 29/70] Exposing /rest endpoint of content-service. --- .../templates/virtualservice.yaml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/charts/content-service/templates/virtualservice.yaml b/charts/content-service/templates/virtualservice.yaml index a584078..cda64d0 100644 --- a/charts/content-service/templates/virtualservice.yaml +++ b/charts/content-service/templates/virtualservice.yaml @@ -17,6 +17,31 @@ spec: - {{ . }} {{- end }} http: + - match: + - uri: + prefix: /rest + rewrite: + uri: / + route: + - destination: + host: {{ template "service.name" . }} + port: + number: {{ .Values.ports.containerPort.http }} + corsPolicy: + allowOrigin: + {{- range .Values.corsPolicy.allowOrigin }} + - {{ . }} + {{- end }} + allowMethods: + {{- range .Values.corsPolicy.allowMethods}} + - {{ . }} + {{- end }} + allowCredentials: {{ .Values.corsPolicy.allowCredentials }} + allowHeaders: + {{- range .Values.corsPolicy.allowHeaders }} + - {{ . }} + {{- end }} + maxAge: {{ .Values.corsPolicy.maxAge }} - match: - uri: prefix: / -- GitLab From 9aa499a5e6a87a735e3015ccff31046566c7cebb Mon Sep 17 00:00:00 2001 From: abotev-intracol <41337656+abotev-intracol@users.noreply.github.com> Date: Fri, 8 Feb 2019 16:44:56 +0200 Subject: [PATCH 30/70] Change storage provider - change storage provider to Local - add /rest in rest endpoints url - add env variable for storage provider --- src/main/resources/application-production.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 47bd9a5..0258c4a 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -36,16 +36,16 @@ media-types: file: upload: - url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net/file/upload} + url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net/rest/file/upload/} job: ttl: 24 # measured in hours. download: - url: ${FILE_DOWNLOAD_URL:https://content.dev-eu.nynja.net/file/download/} + url: ${FILE_DOWNLOAD_URL:https://content.dev-eu.nynja.net/rest/file/download/} storage: - provider: GOOGLE + provider: ${STORAGE_PROVIDER:LOCAL} local: - location: ${LOCAL_STORAGE_LOCATION:/src/main/resources} + location: ${LOCAL_STORAGE_LOCATION:/opt/nynja/} google: uri: ${GOOGLE_STORAGE_URI:https://storage.googleapis.com} bucket: ${GOOGLE_STORAGE_BUCKET:content-service-dev} -- GitLab From f937f0312e5b1ea172a5c9dbd011c3286774f32a Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Thu, 7 Feb 2019 18:43:43 +0200 Subject: [PATCH 31/70] NY-6915 Chunk size on upload - fix logic of content-range for Google storage upload; Signed-off-by: abotev-intracol --- .../content/file/storage/StorageProvider.java | 2 +- .../storage/impl/GoogleStorageProvider.java | 23 ++++++++++++------- .../storage/impl/LocalStorageProvider.java | 2 +- .../file/upload/rest/FileUploadService.java | 2 +- .../grpc/services/ContentServiceImpl.java | 6 ++--- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index 6264d3f..b367253 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -13,7 +13,7 @@ public interface StorageProvider { public String initialize(String fileName) throws Exception; - public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws Exception; + public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk, int lastUploadedByte) throws Exception; public void close(String fileLocation) throws IOException; diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 6d0dc0a..0ab6873 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -81,16 +81,16 @@ public class GoogleStorageProvider implements StorageProvider { } @Override - public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws Exception { + public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk, int lastUploadedByte) throws Exception { logger.debug("Writing partId: {} with length {} to: {}", chunkCount, data.length, fileLocation); - --chunkCount; // It's necessary because chunkCount starts from 1 not from 0 + try (InputStream inputStream = new ByteBufInputStream(Unpooled.wrappedBuffer(data))) { - int length = Math.min(Unpooled.wrappedBuffer(data).readableBytes(), - storageConfiguration.getUploadChunkSize()); + /* int length = Math.min(Unpooled.wrappedBuffer(data).readableBytes(), + storageConfiguration.getUploadChunkSize());*/ HttpContent content = new InputStreamContent(null, inputStream); String URI = fileLocation; try { - buildAndExecuteUpdateRequest(URI, content, chunkCount, length, isFinalChunk); + buildAndExecuteUpdateRequest(URI, content, chunkCount, data.length, isFinalChunk, lastUploadedByte); logger.debug("Success upload to location: {}" + fileLocation); } catch (HttpResponseException e) { if (e.getMessage().equals("308 Resume Incomplete")) { @@ -142,16 +142,23 @@ public class GoogleStorageProvider implements StorageProvider { } private HttpResponse buildAndExecuteUpdateRequest(String URI, HttpContent content, int chunkCount, int length, - boolean isFinalChunk) throws IOException { + boolean isFinalChunk, int lastUploadedByte) throws IOException { GenericUrl url = new GenericUrl(URI); HttpRequest req = requestFactory.buildPutRequest(url, content); - int offset = chunkCount * storageConfiguration.getUploadChunkSize(); + /* int offset = chunkCount * storageConfiguration.getUploadChunkSize(); long limit = offset + length; HttpHeaders headers = new HttpHeaders(); headers.setContentLength((long) length); headers.setContentRange( - "bytes " + (length == 0 ? "*" : offset + "-" + (limit - 1)) + (isFinalChunk ? "/" + limit : "/*")); + "bytes " + (length == 0 ? "*" : offset + "-" + (limit - 1)) + (isFinalChunk ? "/" + limit : "/*"));*/ + + long limit = lastUploadedByte + length; + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength((long) length); + headers.setContentRange( + "bytes " + (lastUploadedByte + "-" + (limit - 1)) + (isFinalChunk ? "/" + limit : "/*")); + req.setHeaders(headers); logger.debug("Executing PUT request to: {}", URI); diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 29c75aa..dedd843 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -44,7 +44,7 @@ public class LocalStorageProvider implements StorageProvider { } @Override - public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws IOException { + public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk, int lastUploadedByte) throws IOException { logger.debug("Appending bytes to file: {}", fileLocation); FileOutputStream output = new FileOutputStream(fileLocation, true); try { diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index f665d10..c9c8e10 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -52,7 +52,7 @@ public class FileUploadService { uploadInfo = initializeUpload(uploadInfo); } - storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last); + storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last, uploadInfo.getLastUploadedByte()); logger.info("Chunk {} was successfully written to output stream {}.", partId, uploadInfo.getFileName()); uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index b301e5b..885d50c 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -173,8 +173,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas try { if (storageProvider == null) { - storageProvider = storageProviderPool - .getStorageProviderByType(provider); + storageProvider = storageProviderPool.getStorageProviderByType(provider); fileName = constructFileName(uploadInfo.getFileName()); fileLocation = storageProvider.initialize(fileName); uploadInfo.setStatus(UploadStatus.IN_PROGRESS); @@ -195,7 +194,8 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas } receivedBytesCount += data.length; chunkCount++; - storageProvider.write(data, fileLocation, chunkCount, (uploadInfo.getFileSize() == receivedBytesCount)); + storageProvider.write(data, fileLocation, chunkCount, (uploadInfo.getFileSize() == receivedBytesCount), + uploadInfo.getLastUploadedByte()); reachedSize += data.length; logger.info("Chunk {} was successfully written to output stream {}.", chunkCount, fileName); } catch (Exception e) { -- GitLab From 1f31db2045fc32249ac7b7df62a71a51b3642667 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 8 Feb 2019 16:56:17 +0200 Subject: [PATCH 32/70] NY-6915 Chunk size on upload - add validation and remove property from code; Signed-off-by: abotev-intracol --- .../file/storage/StorageConfiguration.java | 15 --------------- .../file/storage/impl/GoogleStorageProvider.java | 9 --------- .../rest/validation/ChunkUploadValidator.java | 10 +++++++++- 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/StorageConfiguration.java b/src/main/java/biz/nynja/content/file/storage/StorageConfiguration.java index 09b3adf..cebddad 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageConfiguration.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageConfiguration.java @@ -16,24 +16,12 @@ public class StorageConfiguration { private final String localStorageLocation; private final String googleStorageURI; private final String googleBucketName; - private final int uploadChunkSize; @Autowired public StorageConfiguration(Environment env) { this.localStorageLocation = env.getRequiredProperty("storage.local.location"); this.googleStorageURI = env.getRequiredProperty("storage.google.uri"); this.googleBucketName = env.getRequiredProperty("storage.google.bucket"); - this.uploadChunkSize = parseProperty(env, "storage.google.upload_chunk_size"); - } - - private int parseProperty(Environment env, String property) throws InternalError { - int intProperty = 0; - try { - intProperty = Integer.parseInt(env.getRequiredProperty(property)); - } catch (NumberFormatException ex) { - throw new InternalError("Integer expected in " + property); - } - return intProperty; } public String getLocalStorageLocation() { @@ -48,7 +36,4 @@ public class StorageConfiguration { return googleBucketName; } - public int getUploadChunkSize() { - return uploadChunkSize; - } } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 0ab6873..577fca5 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -85,8 +85,6 @@ public class GoogleStorageProvider implements StorageProvider { logger.debug("Writing partId: {} with length {} to: {}", chunkCount, data.length, fileLocation); try (InputStream inputStream = new ByteBufInputStream(Unpooled.wrappedBuffer(data))) { - /* int length = Math.min(Unpooled.wrappedBuffer(data).readableBytes(), - storageConfiguration.getUploadChunkSize());*/ HttpContent content = new InputStreamContent(null, inputStream); String URI = fileLocation; try { @@ -146,13 +144,6 @@ public class GoogleStorageProvider implements StorageProvider { GenericUrl url = new GenericUrl(URI); HttpRequest req = requestFactory.buildPutRequest(url, content); - /* int offset = chunkCount * storageConfiguration.getUploadChunkSize(); - long limit = offset + length; - HttpHeaders headers = new HttpHeaders(); - headers.setContentLength((long) length); - headers.setContentRange( - "bytes " + (length == 0 ? "*" : offset + "-" + (limit - 1)) + (isFinalChunk ? "/" + limit : "/*"));*/ - long limit = lastUploadedByte + length; HttpHeaders headers = new HttpHeaders(); headers.setContentLength((long) length); diff --git a/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java index 22e0594..3b3fde9 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java @@ -36,9 +36,17 @@ public class ChunkUploadValidator { "You are trying to upload wrong data chunk. Last uploaded chunk " + uploadInfo.getPartId())); } if (data.length <= 0) { - logger.error("Recieved empty data chuck for jobId {} with associated token: {}", jobId, token); + logger.error("Recieved empty data chunk for jobId {} with associated token: {}", jobId, token); return Optional.of(new ValidationResult("Missing chunk data.")); } + if (data.length % 262144 != 0) { + logger.error("Recieved data chunk with size not mupltiple of 256KB(262144 bytes) for jobId {} with associated token: {}", jobId, token); + return Optional.of(new ValidationResult("Data is not multiple of 256KB.")); + } + if (last == false && data.length < 262144) { + logger.error("Recieved data chunk with size smaller than 256KB(262144 bytes) for jobId {} with associated token: {}", jobId, token); + return Optional.of(new ValidationResult("Data smaller than 256KB is allowed only in last chunk.")); + } String sizeValidationResult = validateReachedFileSize(uploadInfo.getLastUploadedByte() + data.length, uploadInfo.getFileSize(), last); -- GitLab From edc25145a3ede6f46bf9a008df361066ca6aa7ee Mon Sep 17 00:00:00 2001 From: abotev-intracol <41337656+abotev-intracol@users.noreply.github.com> Date: Mon, 11 Feb 2019 12:53:28 +0200 Subject: [PATCH 33/70] Increase ttl of upload token for qa purpose --- src/main/resources/application-production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 0258c4a..6c9a2e4 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -20,7 +20,7 @@ spring: token: encryptdecrypt: key: someKeyUsedForEncryptionAndDecryption - time_to_live: 600 # measured in seconds + time_to_live: 36000 # measured in seconds max_file_size: 1500000000 #measured in bytes (B) max_upload_retries: 3 -- GitLab From a2ecaf78f9ef5f0b959710de4b2aceefa82d1783 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 12 Feb 2019 14:14:54 +0200 Subject: [PATCH 34/70] NY-6931 [BE] no finallize upload - missed check if chunk is last in validation; Signed-off-by: abotev-intracol --- .../file/upload/rest/validation/ChunkUploadValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java index 3b3fde9..f814dd9 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java @@ -39,7 +39,7 @@ public class ChunkUploadValidator { logger.error("Recieved empty data chunk for jobId {} with associated token: {}", jobId, token); return Optional.of(new ValidationResult("Missing chunk data.")); } - if (data.length % 262144 != 0) { + if (last == false && data.length % 262144 != 0) { logger.error("Recieved data chunk with size not mupltiple of 256KB(262144 bytes) for jobId {} with associated token: {}", jobId, token); return Optional.of(new ValidationResult("Data is not multiple of 256KB.")); } -- GitLab From d849ccc592dbc46e6de834c3837c6e243d889e72 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 12 Feb 2019 16:07:39 +0200 Subject: [PATCH 35/70] NY-6932 [BE] Download link missing in response - fix bug; Signed-off-by: abotev-intracol --- .../biz/nynja/content/file/upload/rest/FileUploadService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index c9c8e10..7b03b85 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -68,9 +68,9 @@ public class FileUploadService { } if (last) { - finalizeUpload(uploadInfo); + String downloadUrl = finalizeUpload(uploadInfo); return new UploadResponse("File upload successfully completed.", UploadStatus.COMPLETED, - uploadInfo.getPartId()); + uploadInfo.getPartId(), downloadUrl); } return new UploadResponse("Data chunk upload successfully completed.", UploadStatus.IN_PROGRESS, -- GitLab From 3a7bc4f713ba9ae95a4cd3749d2c2caefff557ea Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Thu, 14 Feb 2019 13:54:09 +0200 Subject: [PATCH 36/70] NY-6938 [BE] content service > uploaded and downloaded file should have same name and extension - remove final encoding and substring random characters from filename; Signed-off-by: abotev-intracol --- .../content/file/download/FileDownloadController.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java index 0ea9039..e3a8978 100644 --- a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java +++ b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java @@ -6,7 +6,6 @@ package biz.nynja.content.file.download; import static biz.nynja.content.core.validation.Validators.util; import java.io.FileNotFoundException; -import java.io.IOException; import java.net.MalformedURLException; import java.util.UUID; @@ -81,11 +80,13 @@ public class FileDownloadController { } if (fileName == null) { fileName = RandomStringUtils.randomAlphabetic(15); + } else { + fileName = fileName.substring(11); } return ResponseEntity.ok().contentType(MediaType.parseMediaType(contentType)) - .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + Base64URL.encode(fileName) + "\"") + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileName + "\"") .body(resource); } + } -- GitLab From 4bc4237f64777fcb3578e2cbe69cc99a0b0139de Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Thu, 14 Feb 2019 14:12:19 +0200 Subject: [PATCH 37/70] NY-6936 [BE] content service > duplications in log file - change log level and remove unnecessary log; Signed-off-by: abotev-intracol --- .../nynja/content/file/upload/token/UploadTokenService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index df9487f..238acc5 100644 --- a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -71,13 +71,12 @@ public class UploadTokenService { try { if (permanent) { uploadStoreRepository.save(uploadInfo); - logger.info("Upload token successfully stored in DB."); + logger.debug("Upload token successfully stored in DB."); } else { uploadStoreRepository.saveWithTtl(uploadInfo, uploadTokenConfiguration.getTokenTimeToLive()); - logger.info("Upload token successfully stored in DB for {} seconds.", + logger.debug("Upload token successfully stored in DB for {} seconds.", uploadTokenConfiguration.getTokenTimeToLive()); } - logger.info("Upload token successfully stored in DB."); return true; } catch (Exception e) { logger.error("Error storing record {} in DB: {}", uploadInfo.toString(), e.getMessage()); -- GitLab From a22b5bbaf7918402f3ae1323149b0e322e4202c0 Mon Sep 17 00:00:00 2001 From: abotev-intracol <41337656+abotev-intracol@users.noreply.github.com> Date: Fri, 15 Feb 2019 09:44:32 +0200 Subject: [PATCH 38/70] Set deleting of old upload jobs on every 1 hour QA purpose --- src/main/resources/application-production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 6c9a2e4..e2d71a1 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -38,7 +38,7 @@ file: upload: url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net/rest/file/upload/} job: - ttl: 24 # measured in hours. + ttl: 1 # measured in hours. download: url: ${FILE_DOWNLOAD_URL:https://content.dev-eu.nynja.net/rest/file/download/} -- GitLab From 235746731ffa611bc0b517ee04c09f5ecf69f246 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Mon, 18 Feb 2019 14:49:06 +0200 Subject: [PATCH 39/70] NY-6954 [BE] Rest > delete > doesn't work correctly - fix bug; Signed-off-by: abotev-intracol --- .../biz/nynja/content/file/upload/rest/JobController.java | 2 +- .../nynja/content/file/upload/token/UploadTokenService.java | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/upload/rest/JobController.java b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java index f815a16..5914049 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/JobController.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java @@ -29,7 +29,7 @@ public class JobController { logger.info("Remove not finished (old) jobs "); Optional> oldJobs = uploadTokenService.getOldUploadJobs(); - if (oldJobs.isPresent()) { + if (oldJobs.isPresent() && !oldJobs.get().isEmpty()) { logger.info("Founded old jobs! Countinue with delete process", oldJobs); uploadTokenService.deleteJobs(oldJobs.get()); return new ResponseEntity("Old jobs successfully deleted!", HttpStatus.NO_CONTENT); diff --git a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index 238acc5..89da274 100644 --- a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; import biz.nynja.content.security.EncryptDecryptService; +import biz.nynja.content.file.upload.models.UploadStatus; import biz.nynja.content.file.upload.models.UploadStore; import biz.nynja.content.file.upload.repositories.UploadStoreRepository; import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; @@ -128,8 +129,11 @@ public class UploadTokenService { int jobTTL = contentServiceConfiguration.getJobTTL(); Instant now = Instant.now(); for (UploadStore job : jobs) { + if(job.getStatus() == UploadStatus.PENDING) { + continue; + } Instant last = Instant.ofEpochMilli(job.getLastUploadedChunkTimestamp()); - long hours = ChronoUnit.HOURS.between(now, last); + long hours = ChronoUnit.HOURS.between(last, now); if (Math.toIntExact(hours) > jobTTL) { oldJobs.add(job); } -- GitLab From 6802048dac9546f7423f9cf18395d078dde97920 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Mon, 18 Feb 2019 19:10:29 +0200 Subject: [PATCH 40/70] NY-6954 [BE] Rest > delete > doesn't work correctly - add dynamic storage reader; Signed-off-by: abotev-intracol --- .../file/upload/models/UploadStore.java | 15 +++++++++ .../file/upload/rest/FileUploadService.java | 16 +++++++--- .../file/upload/rest/JobController.java | 32 +++++++++++++++---- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java index b902848..892f2c2 100644 --- a/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java @@ -30,6 +30,7 @@ public class UploadStore { private int retries; private int partId; private int lastUploadedByte; + private String storage; private String uploadLocation; private Long lastUploadedChunkTimestamp; @@ -168,6 +169,14 @@ public class UploadStore { this.lastUploadedByte = lastUploadedByte; } + public String getStorage() { + return storage; + } + + public void setStorage(String storage) { + this.storage = storage; + } + public String getUploadLocation() { return uploadLocation; } @@ -200,6 +209,7 @@ public class UploadStore { result = prime * result + retries; result = prime * result + ((status == null) ? 0 : status.hashCode()); result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); + result = prime * result + ((storage == null) ? 0: storage.hashCode()); result = prime * result + ((uploadLocation == null) ? 0 : uploadLocation.hashCode()); result = prime * result + ((uploadToken == null) ? 0 : uploadToken.hashCode()); result = prime * result + ((uploadUrl == null) ? 0 : uploadUrl.hashCode()); @@ -261,6 +271,11 @@ public class UploadStore { return false; } else if (!tokenExpirationTime.equals(other.tokenExpirationTime)) return false; + if (storage == null) { + if (other.storage != null) + return false; + } else if (!storage.equals(other.storage)) + return false; if (uploadLocation == null) { if (other.uploadLocation != null) return false; diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index 7b03b85..39b046f 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -14,6 +14,7 @@ import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.FileMetadataService; import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageProvider; import biz.nynja.content.file.storage.StorageProviderPool; import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; @@ -34,6 +35,7 @@ public class FileUploadService { private UploadTokenService uploadTokenService; private FileMetadataService fileMetadataService; private ContentServiceConfiguration contentServiceConfiguration; + private StorageProviderPool storageProviderPool; @Autowired public FileUploadService(StorageProviderPool storageProviderPool, UploadTokenService uploadTokenService, @@ -41,6 +43,7 @@ public class FileUploadService { this.uploadTokenService = uploadTokenService; this.fileMetadataService = fileMetadataService; this.contentServiceConfiguration = contentServiceConfiguration; + this.storageProviderPool = storageProviderPool; this.storageProvider = storageProviderPool .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); } @@ -52,7 +55,8 @@ public class FileUploadService { uploadInfo = initializeUpload(uploadInfo); } - storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last, uploadInfo.getLastUploadedByte()); + storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last, + uploadInfo.getLastUploadedByte()); logger.info("Chunk {} was successfully written to output stream {}.", partId, uploadInfo.getFileName()); uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); @@ -85,7 +89,7 @@ public class FileUploadService { String uploadLocation = storageProvider.initialize(encodedFileName); uploadInfo.setUploadLocation(uploadLocation); - + uploadInfo.setStorage(storageProvider.getProviderType().toString()); return uploadInfo; } @@ -118,10 +122,12 @@ public class FileUploadService { public boolean cancelUploadProcess(UploadStore uploadInfo) { try { - this.storageProvider.close(uploadInfo.getUploadLocation()); + StorageProvider usedStorageProvider = storageProviderPool + .getStorageProviderByType(Provider.valueOf(uploadInfo.getStorage())); + usedStorageProvider.close(uploadInfo.getUploadLocation()); } catch (IOException e) { - logger.error("Error canceling upload job: {}", e.getMessage()); - logger.debug("Error canceling upload job: {}", e.getCause()); + logger.error("Error canceling upload job with id: {}, {}", uploadInfo.getJobId(), e.getMessage()); + logger.debug("Error canceling upload job with id: {}, {}", uploadInfo.getJobId(), e.getCause()); return false; } uploadTokenService.deleteToken(uploadInfo.getUploadToken()); diff --git a/src/main/java/biz/nynja/content/file/upload/rest/JobController.java b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java index 5914049..bd4139a 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/JobController.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java @@ -1,7 +1,9 @@ package biz.nynja.content.file.upload.rest; +import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,22 +22,40 @@ public class JobController { private UploadTokenService uploadTokenService; - public JobController(UploadTokenService uploadTokenService) { + private FileUploadService fileUploadService; + + public JobController(UploadTokenService uploadTokenService, FileUploadService fileUploadService) { this.uploadTokenService = uploadTokenService; + this.fileUploadService = fileUploadService; } @DeleteMapping("/jobs/old") - public ResponseEntity removeOldJobs() { + public ResponseEntity> removeOldJobs() { logger.info("Remove not finished (old) jobs "); Optional> oldJobs = uploadTokenService.getOldUploadJobs(); + HashMap statuses = null; if (oldJobs.isPresent() && !oldJobs.get().isEmpty()) { - logger.info("Founded old jobs! Countinue with delete process", oldJobs); - uploadTokenService.deleteJobs(oldJobs.get()); - return new ResponseEntity("Old jobs successfully deleted!", HttpStatus.NO_CONTENT); + logger.info("Founded old jobs! Continue with delete process", oldJobs); + statuses = removeJobs(oldJobs.get()); + return new ResponseEntity>(statuses, HttpStatus.OK); } else { logger.info("Old jobs didn't found!"); - return new ResponseEntity("Old jobs didn't found!", HttpStatus.NO_CONTENT); + statuses = new HashMap<>(); + statuses.put("Old jobs didn't found!", null); + return new ResponseEntity>(statuses, HttpStatus.OK); } } + + private HashMap removeJobs(List oldJobs) { + HashMap statuses = new HashMap<>(); + oldJobs.forEach(oldJob -> { + if(fileUploadService.cancelUploadProcess(oldJob)) { + statuses.put("Successfully deleted job: ", oldJob.getJobId()); + }else { + statuses.put("Failed deleted job", oldJob.getJobId()); + } + }); + return statuses; + } } -- GitLab From 0001536c58747ba7dbf7222bd9a449ca63f45124 Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Wed, 20 Feb 2019 16:15:17 +0200 Subject: [PATCH 41/70] Add gcloud service account file. --- charts/content-service/templates/deployment.yaml | 10 ++++++++++ charts/content-service/values.yaml | 13 +++++++++++++ releases/dev/content-service.yaml | 13 +++++++++++++ 3 files changed, 36 insertions(+) diff --git a/charts/content-service/templates/deployment.yaml b/charts/content-service/templates/deployment.yaml index bf29d0b..090eeaf 100644 --- a/charts/content-service/templates/deployment.yaml +++ b/charts/content-service/templates/deployment.yaml @@ -58,6 +58,16 @@ spec: value: {{ .Values.ports.containerPort.http | quote }} - name: GRPC_SERVER_PORT value: {{ .Values.ports.containerPort.grpc | quote }} +{{ if .Values.extra_vars -}} +{{ toYaml .Values.extra_vars | indent 8 }} +{{- end }} + volumes: + - name: service-account + secret: + secretName: service-account + volumeMounts: + - name: service-account + mountPath: /opt/nynja/config resources: {{ toYaml .Values.resources | indent 12 }} {{- with .Values.nodeSelector }} diff --git a/charts/content-service/values.yaml b/charts/content-service/values.yaml index a1a770d..dc61f6f 100644 --- a/charts/content-service/values.yaml +++ b/charts/content-service/values.yaml @@ -36,3 +36,16 @@ corsPolicy: allowHeaders: maxAge: +extra_vars: + - name: FILE_UPLOAD_URL + value: https://content.dev-eu.nynja.net/file/upload + - name: FILE_DOWNLOAD_URL + value: https://content.dev-eu.nynja.net/file/download/ + - name: LOCAL_STORAGE_LOCATION + value: /src/main/resources + - name: GOOGLE_STORAGE_URI + value: https://storage.googleapis.com + - name: GOOGLE_STORAGE_BUCKET + value: content-service-dev + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /opt/nynja/application-credentials.json diff --git a/releases/dev/content-service.yaml b/releases/dev/content-service.yaml index 40887f1..745dc30 100644 --- a/releases/dev/content-service.yaml +++ b/releases/dev/content-service.yaml @@ -53,3 +53,16 @@ spec: - x-grpc-web maxAge: "600s" + extra_vars: + - name: FILE_UPLOAD_URL + value: https://content.dev-eu.nynja.net/file/upload + - name: FILE_DOWNLOAD_URL + value: https://content.dev-eu.nynja.net/file/download/ + - name: LOCAL_STORAGE_LOCATION + value: /src/main/resources + - name: GOOGLE_STORAGE_URI + value: https://storage.googleapis.com + - name: GOOGLE_STORAGE_BUCKET + value: content-service-dev + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /opt/nynja/config/application-credentials.json -- GitLab From 4e43b49a9a5e5ec225918f0b5b29c9e8a6604c06 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 20 Feb 2019 16:36:51 +0200 Subject: [PATCH 42/70] NYNJA-6994 [BE] content service > Delete endpoint issue - fix response and logs; Signed-off-by: abotev-intracol --- .../file/upload/rest/JobController.java | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/upload/rest/JobController.java b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java index bd4139a..afb7bd0 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/JobController.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java @@ -1,9 +1,7 @@ package biz.nynja.content.file.upload.rest; -import java.util.HashMap; import java.util.List; import java.util.Optional; -import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,32 +28,39 @@ public class JobController { } @DeleteMapping("/jobs/old") - public ResponseEntity> removeOldJobs() { + public ResponseEntity removeOldJobs() { logger.info("Remove not finished (old) jobs "); Optional> oldJobs = uploadTokenService.getOldUploadJobs(); - HashMap statuses = null; + int counter = 0; if (oldJobs.isPresent() && !oldJobs.get().isEmpty()) { logger.info("Founded old jobs! Continue with delete process", oldJobs); - statuses = removeJobs(oldJobs.get()); - return new ResponseEntity>(statuses, HttpStatus.OK); + List oldJobsList = oldJobs.get(); + counter = removeJobs(oldJobsList); + logger.info("Delete process finish with {} of {} successfully deleted jobs.", counter, oldJobsList.size()); + if (counter == oldJobsList.size()) { + return new ResponseEntity("Successfully deleted old jobs.", HttpStatus.OK); + } else { + return new ResponseEntity( + "Delete process finish with " + (oldJobsList.size() - counter) + " failed jobs.", + HttpStatus.INTERNAL_SERVER_ERROR); + } } else { logger.info("Old jobs didn't found!"); - statuses = new HashMap<>(); - statuses.put("Old jobs didn't found!", null); - return new ResponseEntity>(statuses, HttpStatus.OK); + return new ResponseEntity("Old jobs didn't found!", HttpStatus.OK); } } - private HashMap removeJobs(List oldJobs) { - HashMap statuses = new HashMap<>(); - oldJobs.forEach(oldJob -> { - if(fileUploadService.cancelUploadProcess(oldJob)) { - statuses.put("Successfully deleted job: ", oldJob.getJobId()); - }else { - statuses.put("Failed deleted job", oldJob.getJobId()); + private int removeJobs(List oldJobs) { + int counter = 0; + for (UploadStore oldJob : oldJobs) { + if (fileUploadService.cancelUploadProcess(oldJob)) { + counter++; + logger.debug("Successfully deleted job: {}", oldJob.getJobId()); + } else { + logger.error("Failed deleted job: {}", oldJob.getJobId()); } - }); - return statuses; + } + return counter; } } -- GitLab From 9915d0cbd8ed8d1513059c60a8a9f03b950cc861 Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Wed, 20 Feb 2019 17:02:53 +0200 Subject: [PATCH 43/70] Fix volume indentation. --- charts/content-service/templates/deployment.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/content-service/templates/deployment.yaml b/charts/content-service/templates/deployment.yaml index 090eeaf..337847d 100644 --- a/charts/content-service/templates/deployment.yaml +++ b/charts/content-service/templates/deployment.yaml @@ -61,10 +61,6 @@ spec: {{ if .Values.extra_vars -}} {{ toYaml .Values.extra_vars | indent 8 }} {{- end }} - volumes: - - name: service-account - secret: - secretName: service-account volumeMounts: - name: service-account mountPath: /opt/nynja/config @@ -82,3 +78,8 @@ spec: tolerations: {{ toYaml . | indent 8 }} {{- end }} + + volumes: + - name: service-account + secret: + secretName: service-account -- GitLab From fe7b53a6c7faa0f5e47284fee70bcc135c10174e Mon Sep 17 00:00:00 2001 From: abotev-intracol <41337656+abotev-intracol@users.noreply.github.com> Date: Thu, 21 Feb 2019 11:40:26 +0200 Subject: [PATCH 44/70] Update production configurations - Change storage provider to GOOGLE --- src/main/resources/application-production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index e2d71a1..782d276 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -43,7 +43,7 @@ file: url: ${FILE_DOWNLOAD_URL:https://content.dev-eu.nynja.net/rest/file/download/} storage: - provider: ${STORAGE_PROVIDER:LOCAL} + provider: ${STORAGE_PROVIDER:GOOGLE} local: location: ${LOCAL_STORAGE_LOCATION:/opt/nynja/} google: -- GitLab From 8b07e0717181c7a37a0f226c0a1492a08c657c0e Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Thu, 21 Feb 2019 17:10:03 +0200 Subject: [PATCH 45/70] NY-7002 [BE] Unit tests - Fix unit tests; Signed-off-by: abotev-intracol --- .../content/file/download/FileDownloadControllerTest.java | 4 ++++ src/test/java/biz/nynja/content/file/upload/Util.java | 2 +- .../java/biz/nynja/content/upload/token/UploadTokenTests.java | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java index ad3ce0c..eb83a8f 100644 --- a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java +++ b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java @@ -28,6 +28,7 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; import biz.nynja.content.file.FileControllersExceptionHandler; +import biz.nynja.content.file.upload.token.UploadTokenService; /** * @author Angel.Botev @@ -54,6 +55,9 @@ public class FileDownloadControllerTest { @Autowired private MockMvc mockMvc; + @MockBean + private UploadTokenService uploadTokenService; + @Test public void testFileDownloadRestOK() throws Exception { org.springframework.core.io.Resource resource = new UrlResource( diff --git a/src/test/java/biz/nynja/content/file/upload/Util.java b/src/test/java/biz/nynja/content/file/upload/Util.java index 47f42fb..9b11cce 100644 --- a/src/test/java/biz/nynja/content/file/upload/Util.java +++ b/src/test/java/biz/nynja/content/file/upload/Util.java @@ -24,7 +24,7 @@ public class Util { public final static String INVALID_UPLOAD_TOKEN = "qwertyuiopasdfghjklzxcvb"; public final static String VALID_UPLOAD_JOBID = "3318f002-5fc1-4aff-bf70-9dbf728c29ca"; public final static String INVALID_UPLOAD_JOBID = "3318f002-5fc1-4aff-bf70-9d"; - public final static int FILE_SIZE = 12345; + public final static int FILE_SIZE = 262144; public final static String FILE_NAME = "somefilename.txt"; public final static byte[] DATA_CHUNK = generateBytes(); public final static String DOWNLOAD_LINK = "https://content.nynja.biz/file/download/kkk"; diff --git a/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java b/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java index 1a6f12d..591ce17 100644 --- a/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java +++ b/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java @@ -31,6 +31,7 @@ import biz.nynja.content.file.upload.repositories.UploadStoreRepository; import biz.nynja.content.file.upload.token.UploadToken; import biz.nynja.content.file.upload.token.UploadTokenService; import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; /** * @author Ralitsa Todorova @@ -60,6 +61,9 @@ public class UploadTokenTests { @Autowired UploadTokenService uploadTokenService; + @MockBean + ContentServiceConfiguration contentServiceConfiguration; + @Test public void testPrepareTokenResponse() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException, -- GitLab From b7aecb57b8ae9f1e0240382bd410bb6d2eb88b7c Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 15 Feb 2019 17:07:22 +0200 Subject: [PATCH 46/70] NY-6949 Create signed Urls for file download - add logic for download file from different storage providers; Signed-off-by: abotev-intracol --- .../file/download/DownloadResponse.java | 53 +++++++++++++++ .../file/download/FileDownloadController.java | 53 +++++---------- .../file/download/FileDownloadService.java | 25 +------ .../file/storage/StorageConfiguration.java | 35 ++++++++++ .../content/file/storage/StorageProvider.java | 9 +++ .../storage/impl/GoogleStorageProvider.java | 65 +++++++++++++++++- .../storage/impl/LocalStorageProvider.java | 67 ++++++++++++++++++- src/main/resources/application-dev.yml | 5 ++ src/main/resources/application-production.yml | 5 ++ 9 files changed, 254 insertions(+), 63 deletions(-) create mode 100644 src/main/java/biz/nynja/content/file/download/DownloadResponse.java diff --git a/src/main/java/biz/nynja/content/file/download/DownloadResponse.java b/src/main/java/biz/nynja/content/file/download/DownloadResponse.java new file mode 100644 index 0000000..9b3f5ed --- /dev/null +++ b/src/main/java/biz/nynja/content/file/download/DownloadResponse.java @@ -0,0 +1,53 @@ +package biz.nynja.content.file.download; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.core.io.Resource; + +public class DownloadResponse { + + private HttpStatus httpStatus; + private String contentType; + private HttpHeaders httpHeaders; + private Resource resource; + + public DownloadResponse(HttpStatus httpStatus, String contentType, HttpHeaders httpHeaders, Resource resource) { + this.httpStatus = httpStatus; + this.contentType = contentType; + this.httpHeaders = httpHeaders; + this.resource = resource; + } + + public HttpStatus getHttpStatus() { + return httpStatus; + } + + public void setHttpStatus(HttpStatus httpStatus) { + this.httpStatus = httpStatus; + } + + public String getContentType() { + return contentType; + } + + public void setContentType(String contentType) { + this.contentType = contentType; + } + + public HttpHeaders getHttpHeaders() { + return httpHeaders; + } + + public void setHttpHeaders(HttpHeaders httpHeaders) { + this.httpHeaders = httpHeaders; + } + + public Resource getResource() { + return resource; + } + + public void setResource(Resource resource) { + this.resource = resource; + } + +} diff --git a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java index e3a8978..f2ab60a 100644 --- a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java +++ b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java @@ -6,17 +6,14 @@ package biz.nynja.content.file.download; import static biz.nynja.content.core.validation.Validators.util; import java.io.FileNotFoundException; -import java.net.MalformedURLException; import java.util.UUID; import javax.servlet.http.HttpServletRequest; -import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; -import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; @@ -24,9 +21,10 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; -import com.nimbusds.jose.util.Base64URL; - -import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.Provider; +import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.storage.StorageProviderPool; /** * @author Angel.Botev @@ -40,12 +38,13 @@ public class FileDownloadController { private static final Logger logger = LoggerFactory.getLogger(FileDownloadController.class); private FileDownloadService fileDownloadService; - private UploadTokenService uploadTokenService; + + private StorageProviderPool storageProviderPool; @Autowired - public FileDownloadController(FileDownloadService fileDownloadService, UploadTokenService uploadTokenService) { + public FileDownloadController(FileDownloadService fileDownloadService, StorageProviderPool storageProvidedPool) { this.fileDownloadService = fileDownloadService; - this.uploadTokenService = uploadTokenService; + this.storageProviderPool = storageProvidedPool; } @RequestMapping(method = RequestMethod.GET, value = "/{fileKey}") @@ -57,36 +56,16 @@ public class FileDownloadController { throw new IllegalArgumentException("Invalid fileKey"); } UUID fileKeyUuid = UUID.fromString(fileKey); - Resource resource = null; - try { - // Load file as Resource - resource = fileDownloadService.getFileContent(fileKeyUuid); - } catch (FileNotFoundException | MalformedURLException e) { - throw new FileNotFoundException("Error downloading file: " + e.getMessage()); - } - // Try to determine file's content type - String contentType = null; - String fileName = null; - try { - fileName = uploadTokenService.decodeFileName(resource.getFile().getName()); - contentType = request.getServletContext().getMimeType(fileName); - } catch (Exception ex) { - logger.info("Could not determine file type."); - } - // Fallback to the default content type if type could not be determined - if (contentType == null) { - contentType = "application/octet-stream"; - } - if (fileName == null) { - fileName = RandomStringUtils.randomAlphabetic(15); - } else { - fileName = fileName.substring(11); - } + FileMetadata fileMetaData = fileDownloadService.getFileInfo(fileKeyUuid); + StorageProvider storageProvider = storageProviderPool + .getStorageProviderByType(Provider.valueOf(fileMetaData.getStorage())); + + DownloadResponse downloadResponse = storageProvider.download(fileMetaData, request); - return ResponseEntity.ok().contentType(MediaType.parseMediaType(contentType)) - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileName + "\"") - .body(resource); + return ResponseEntity.status(downloadResponse.getHttpStatus()) + .contentType(MediaType.parseMediaType(downloadResponse.getContentType())) + .headers(downloadResponse.getHttpHeaders()).body(downloadResponse.getResource()); } } diff --git a/src/main/java/biz/nynja/content/file/download/FileDownloadService.java b/src/main/java/biz/nynja/content/file/download/FileDownloadService.java index 5af0ee5..d8e105d 100644 --- a/src/main/java/biz/nynja/content/file/download/FileDownloadService.java +++ b/src/main/java/biz/nynja/content/file/download/FileDownloadService.java @@ -3,17 +3,12 @@ */ package biz.nynja.content.file.download; -import java.io.File; -import java.io.FileNotFoundException; -import java.net.MalformedURLException; import java.util.MissingResourceException; import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.Resource; -import org.springframework.core.io.UrlResource; import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.dto.FileMetadata; @@ -35,30 +30,14 @@ public class FileDownloadService { this.fileMetadataReposiory = fileMetadataReposiory; } - public Resource getFileContent(UUID key) throws FileNotFoundException, MalformedURLException { - String fileName = getFileName(key); - return loadFileAsResource(fileName); - } - - private String getFileName(UUID fileKey) { + public FileMetadata getFileInfo(UUID fileKey) { FileMetadata fileMetadata = fileMetadataReposiory.findByKey(fileKey); if (fileMetadata == null) { logger.error("No record with key {} found in DB.", fileKey); throw new MissingResourceException("No inforamtion found for the given key.", FileMetadata.class.getName(), fileKey.toString()); } - return fileMetadata.getFileUrl(); - } - - private Resource loadFileAsResource(String fileName) throws FileNotFoundException, MalformedURLException { - Resource resource = new UrlResource(new File(fileName).toURI()); - if (resource.exists()) { - logger.debug("File {} found successfully!", fileName); - return resource; - } else { - logger.error("File {} not found!", fileName); - throw new FileNotFoundException("File not found!"); - } + return fileMetadata; } } diff --git a/src/main/java/biz/nynja/content/file/storage/StorageConfiguration.java b/src/main/java/biz/nynja/content/file/storage/StorageConfiguration.java index cebddad..886c8b9 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageConfiguration.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageConfiguration.java @@ -16,12 +16,31 @@ public class StorageConfiguration { private final String localStorageLocation; private final String googleStorageURI; private final String googleBucketName; + private final String signKeyName; + private final String signKey; + private final int signedUrlTTL; + private final String cdnURI; @Autowired public StorageConfiguration(Environment env) { this.localStorageLocation = env.getRequiredProperty("storage.local.location"); this.googleStorageURI = env.getRequiredProperty("storage.google.uri"); this.googleBucketName = env.getRequiredProperty("storage.google.bucket"); + this.signKeyName = env.getRequiredProperty("storage.google.sign_url.key_name"); + this.signKey = env.getRequiredProperty("storage.google.sign_url.key"); + this.signedUrlTTL = parseProperty(env, "storage.google.sign_url.ttl"); + this.cdnURI = env.getRequiredProperty("storage.google.sign_url.cdn_uri"); + } + + private int parseProperty(Environment env, String property) throws InternalError { + int intProperty = 0; + try { + intProperty = Integer.parseInt(env.getRequiredProperty(property)); + } catch (NumberFormatException ex) { + throw new InternalError("Integer expected in " + property); + } + return intProperty; + } public String getLocalStorageLocation() { @@ -36,4 +55,20 @@ public class StorageConfiguration { return googleBucketName; } + public String getSignKeyName() { + return signKeyName; + } + + public String getSignKey() { + return signKey; + } + + public int getSignedUrlTTL() { + return signedUrlTTL; + } + + public String getCdnURI() { + return cdnURI; + } + } diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index b367253..a57c6ff 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -3,8 +3,14 @@ */ package biz.nynja.content.file.storage; +import java.io.FileNotFoundException; import java.io.IOException; +import javax.servlet.http.HttpServletRequest; + +import biz.nynja.content.file.download.DownloadResponse; +import biz.nynja.content.file.metadata.dto.FileMetadata; + /** * @author Angel.Botev * @@ -20,4 +26,7 @@ public interface StorageProvider { public String getFileUrl(String filename); public Provider getProviderType(); + + public DownloadResponse download(FileMetadata fileMetaData, HttpServletRequest request) throws FileNotFoundException; + } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 577fca5..787cc0b 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -5,13 +5,26 @@ package biz.nynja.content.file.storage.impl; import java.io.IOException; import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Paths; import java.security.GeneralSecurityException; +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.Calendar; import java.util.Collections; +import java.util.Date; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import javax.servlet.http.HttpServletRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; @@ -28,6 +41,8 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.InputStreamContent; import com.google.api.services.storage.StorageScopes; +import biz.nynja.content.file.download.DownloadResponse; +import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageConfiguration; import biz.nynja.content.file.storage.StorageProvider; @@ -81,7 +96,8 @@ public class GoogleStorageProvider implements StorageProvider { } @Override - public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk, int lastUploadedByte) throws Exception { + public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk, int lastUploadedByte) + throws Exception { logger.debug("Writing partId: {} with length {} to: {}", chunkCount, data.length, fileLocation); try (InputStream inputStream = new ByteBufInputStream(Unpooled.wrappedBuffer(data))) { @@ -150,7 +166,6 @@ public class GoogleStorageProvider implements StorageProvider { headers.setContentRange( "bytes " + (lastUploadedByte + "-" + (limit - 1)) + (isFinalChunk ? "/" + limit : "/*")); - req.setHeaders(headers); logger.debug("Executing PUT request to: {}", URI); // Execute request @@ -170,4 +185,50 @@ public class GoogleStorageProvider implements StorageProvider { public Provider getProviderType() { return providerType; } + + @Override + public DownloadResponse download(FileMetadata fileMetaData, HttpServletRequest request) { + Calendar cal = Calendar.getInstance(); + cal.setTime(new Date()); + cal.add(Calendar.SECOND, storageConfiguration.getSignedUrlTTL()); // minus number would decrement the time + URI signedUrlURI = null; + String url = null; + try { + url = storageConfiguration.getCdnURI() + "/" + fileMetaData.getFileName(); + String signedUrl = signUrl(url, storageConfiguration.getSignKey().getBytes(), + storageConfiguration.getSignKeyName(), cal.getTime()); + signedUrlURI = new URI(signedUrl); + + } catch (InvalidKeyException | NoSuchAlgorithmException | URISyntaxException e) { + logger.debug("Error signing url: {}, {}", url, e.getStackTrace()); + throw new InternalError("Error downloading file!"); + } + + org.springframework.http.HttpHeaders httpHeaders = new org.springframework.http.HttpHeaders(); + httpHeaders.setLocation(signedUrlURI); + + return new DownloadResponse(HttpStatus.PERMANENT_REDIRECT, "text/plain", httpHeaders, null); + } + + private String signUrl(String url, byte[] key, String keyName, Date expirationTime) + throws InvalidKeyException, NoSuchAlgorithmException { + + final long unixTime = expirationTime.getTime() / 1000; + + String urlToSign = url + (url.contains("?") ? "&" : "?") + "Expires=" + unixTime + "&KeyName=" + keyName; + + String encoded = getSignature(key, urlToSign); + return urlToSign + "&Signature=" + encoded; + } + + private String getSignature(byte[] privateKey, String input) throws InvalidKeyException, NoSuchAlgorithmException { + + final String algorithm = "HmacSHA1"; + final int offset = 0; + Key key = new SecretKeySpec(privateKey, offset, privateKey.length, algorithm); + Mac mac = Mac.getInstance(algorithm); + mac.init(key); + return Base64.getUrlEncoder().encodeToString(mac.doFinal(input.getBytes())); + } + } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index dedd843..ba89c70 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -7,16 +7,27 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.net.MalformedURLException; import java.nio.file.Files; import java.nio.file.Paths; +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.io.Resource; +import org.springframework.core.io.UrlResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import biz.nynja.content.file.download.DownloadResponse; +import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageConfiguration; import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.upload.token.UploadTokenService; /** * @author Ralitsa Todorova @@ -32,8 +43,11 @@ public class LocalStorageProvider implements StorageProvider { private StorageConfiguration storageConfiguration; - public LocalStorageProvider(StorageConfiguration storageConfiguration) { + private UploadTokenService uploadTokenService; + + public LocalStorageProvider(StorageConfiguration storageConfiguration, UploadTokenService uploadTokenService) { this.storageConfiguration = storageConfiguration; + this.uploadTokenService = uploadTokenService; } @Override @@ -76,4 +90,55 @@ public class LocalStorageProvider implements StorageProvider { return providerType; } + @Override + public DownloadResponse download(FileMetadata fileMetaData, HttpServletRequest request) throws FileNotFoundException { + Resource resource = null; + try { + // Load file as Resource + resource = getFileContent(fileMetaData); + } catch (FileNotFoundException | MalformedURLException e) { + throw new FileNotFoundException("Error downloading file: " + e.getMessage()); + } + // Try to determine file's content type + String contentType = null; + String fileName = null; + try { + fileName = uploadTokenService.decodeFileName(resource.getFile().getName()); + contentType = request.getServletContext().getMimeType(fileName); + } catch (Exception ex) { + logger.info("Could not determine file type."); + } + + // Fallback to the default content type if type could not be determined + if (contentType == null) { + contentType = "application/octet-stream"; + } + if (fileName == null) { + fileName = RandomStringUtils.randomAlphabetic(15); + } else { + fileName = fileName.substring(11); + } + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileName + "\""); + + return new DownloadResponse(HttpStatus.OK, contentType, httpHeaders, resource); + } + + private Resource getFileContent(FileMetadata fileMetaData) throws FileNotFoundException, MalformedURLException { + String fileName = fileMetaData.getFileUrl(); + return loadFileAsResource(fileName); + } + + private Resource loadFileAsResource(String fileName) throws FileNotFoundException, MalformedURLException { + Resource resource = new UrlResource(new File(fileName).toURI()); + if (resource.exists()) { + logger.debug("File {} found successfully!", fileName); + return resource; + } else { + logger.error("File {} not found!", fileName); + throw new FileNotFoundException("File not found!"); + } + } + } diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index b1f98b2..945a5bf 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -50,6 +50,11 @@ storage: uri: https://storage.googleapis.com bucket: content-service-dev upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + sign_url: + key_name: content-service-dev-key + key: dsankldmsakdmkalsdmksa== + ttl: 86400 # measured in seconds. + cdn_uri: http://35.244.165.21 # To enable colors in Eclipse: diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 782d276..55139b0 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -50,6 +50,11 @@ storage: uri: ${GOOGLE_STORAGE_URI:https://storage.googleapis.com} bucket: ${GOOGLE_STORAGE_BUCKET:content-service-dev} upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + sign_url: + key_name: ${SIGN_URL_KEY_NAME:content-service-dev-key} + key: ${SIGN_URL_KEY:dsankldmsakdmkalsdmksa==} + ttl: ${SIGN_URL_TTL:86400} # measured in seconds. + cdn_uri: ${SIGN_URL_CDN_URI:http://35.244.165.21} # To enable colors in Eclipse: # spring.output.ansi.enabled=ALWAYS and in eclipse -- GitLab From c91fbc92880b83b5c2be857f8996a36b9ac00fd6 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 19 Feb 2019 19:21:45 +0200 Subject: [PATCH 47/70] NY-6949 Create signed Urls for file download - fix sign key decoding; Signed-off-by: abotev-intracol --- .../nynja/content/file/storage/impl/GoogleStorageProvider.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 787cc0b..a687ec2 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -195,7 +195,8 @@ public class GoogleStorageProvider implements StorageProvider { String url = null; try { url = storageConfiguration.getCdnURI() + "/" + fileMetaData.getFileName(); - String signedUrl = signUrl(url, storageConfiguration.getSignKey().getBytes(), + byte[] keyBytes = Base64.getUrlDecoder().decode(storageConfiguration.getSignKey()); + String signedUrl = signUrl(url, keyBytes, storageConfiguration.getSignKeyName(), cal.getTime()); signedUrlURI = new URI(signedUrl); -- GitLab From baf516db900c503921922d8b3c3cd3ddff98ce04 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 20 Feb 2019 14:49:56 +0200 Subject: [PATCH 48/70] NY-6949 Create signed Urls for file download - fix rest download controller unit tests; Signed-off-by: abotev-intracol --- .../download/FileDownloadControllerTest.java | 80 ++++++++++++++++--- .../file/download/FileDownloadUtil.java | 50 ++++++++++++ ...w0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULmZpbGU=} | 0 ...w0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULnR4dA==} | 0 4 files changed, 117 insertions(+), 13 deletions(-) create mode 100644 src/test/java/biz/nynja/content/file/download/FileDownloadUtil.java rename src/test/resources/{ONLY_TEST.file => 0LDQsNCw0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULmZpbGU=} (100%) rename src/test/resources/{ONLY_TEST.txt => 0LDQsNCw0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULnR4dA==} (100%) diff --git a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java index eb83a8f..6e060c9 100644 --- a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java +++ b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java @@ -3,13 +3,16 @@ */ package biz.nynja.content.file.download; +import static biz.nynja.content.file.download.FileDownloadUtil.FILE_STORAGE_LOCATION; +import static biz.nynja.content.file.download.FileDownloadUtil.INVALID_UUID; +import static biz.nynja.content.file.download.FileDownloadUtil.TEXT_FILE_EXISTING; +import static biz.nynja.content.file.download.FileDownloadUtil.VALID_UUID; import static org.mockito.BDDMockito.given; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import java.io.File; -import java.io.FileNotFoundException; import java.util.UUID; import javax.annotation.Resource; @@ -17,10 +20,13 @@ import javax.annotation.Resource; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.core.io.UrlResource; +import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; +import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; @@ -28,6 +34,12 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; import biz.nynja.content.file.FileControllersExceptionHandler; +import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.Provider; +import biz.nynja.content.file.storage.StorageConfiguration; +import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.storage.StorageProviderPool; +import biz.nynja.content.file.storage.impl.LocalStorageProvider; import biz.nynja.content.file.upload.token.UploadTokenService; /** @@ -36,22 +48,40 @@ import biz.nynja.content.file.upload.token.UploadTokenService; */ @RunWith(SpringRunner.class) @WebMvcTest(FileDownloadController.class) -@ContextConfiguration(classes = { FileDownloadController.class, FileControllersExceptionHandler.class }) +@ContextConfiguration( + classes = { FileDownloadController.class, FileControllersExceptionHandler.class, FileDownloadUtil.class }) @ActiveProfiles("dev") public class FileDownloadControllerTest { - private final static String FILE_STORAGE_LOCATION = "src/test/resources/"; - private final static String VALID_UUID = "d397c291-4d37-49dc-8d55-f6b08ab34fb2"; - private final static String INVALID_UUID = "d397c291-4d37-49dc-8d55-f6b08ab"; - private final static String TEXT_FILE_EXISTING = "ONLY_TEST.txt"; - private final static String PPTX_FILE_EXISTING = "ONLY_TEST.file"; - @MockBean private FileDownloadService fileDownloadService; @Resource private WebApplicationContext wac; + @Autowired + @Qualifier("fileMetaDataExisting") + private FileMetadata fileMetaData; + + @Autowired + @Qualifier("fileMetaDataNotFound") + private FileMetadata fileMetaDataNotFound; + + @MockBean + private StorageConfiguration storageConfiguration; + + @MockBean + private UploadTokenService uploadTokenService; + + @MockBean + private LocalStorageProvider localStorageProvider; + + @MockBean + private StorageProviderPool storageProviderPool; + + @MockBean + private MockHttpServletRequest request; + @Autowired private MockMvc mockMvc; @@ -60,9 +90,20 @@ public class FileDownloadControllerTest { @Test public void testFileDownloadRestOK() throws Exception { + org.springframework.core.io.Resource resource = new UrlResource( new File(FILE_STORAGE_LOCATION + TEXT_FILE_EXISTING).toURI()); - given(fileDownloadService.getFileContent(UUID.fromString(VALID_UUID))).willReturn(resource); + given(fileDownloadService.getFileInfo(UUID.fromString(VALID_UUID))).willReturn(fileMetaData); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileMetaData.getFileName() + "\""); + + given(uploadTokenService.decodeFileName(resource.getFile().getName())).willReturn("aaaaaaaaaa_ONLY_TEST.txt"); + StorageProvider storageProvider = new LocalStorageProvider(storageConfiguration, uploadTokenService); + + given(storageProviderPool.getStorageProviderByType(Provider.valueOf(fileMetaData.getStorage()))) + .willReturn(storageProvider); + mockMvc.perform(get("/file/download/" + VALID_UUID)).andExpect(status().isOk()) .andExpect(content().contentType(MediaType.TEXT_PLAIN)); } @@ -74,15 +115,28 @@ public class FileDownloadControllerTest { @Test public void testFileDownloadRestNotFound() throws Exception { - given(fileDownloadService.getFileContent(UUID.fromString(VALID_UUID))).willThrow(new FileNotFoundException()); + given(fileDownloadService.getFileInfo(UUID.fromString(VALID_UUID))).willReturn(fileMetaDataNotFound); + + StorageProvider storageProvider = new LocalStorageProvider(storageConfiguration, uploadTokenService); + + given(storageProviderPool.getStorageProviderByType(Provider.valueOf(fileMetaData.getStorage()))) + .willReturn(storageProvider); + mockMvc.perform(get("/file/download/" + VALID_UUID)).andExpect(status().isNotFound()); } @Test public void testFileDownloadOctetStreamContent() throws Exception { - org.springframework.core.io.Resource resource = new UrlResource( - new File(FILE_STORAGE_LOCATION + PPTX_FILE_EXISTING).toURI()); - given(fileDownloadService.getFileContent(UUID.fromString(VALID_UUID))).willReturn(resource); + given(fileDownloadService.getFileInfo(UUID.fromString(VALID_UUID))).willReturn(fileMetaData); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileMetaData.getFileName() + "\""); + + StorageProvider storageProvider = new LocalStorageProvider(storageConfiguration, uploadTokenService); + + given(storageProviderPool.getStorageProviderByType(Provider.valueOf(fileMetaData.getStorage()))) + .willReturn(storageProvider); + mockMvc.perform(get("/file/download/" + VALID_UUID)).andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM)); } diff --git a/src/test/java/biz/nynja/content/file/download/FileDownloadUtil.java b/src/test/java/biz/nynja/content/file/download/FileDownloadUtil.java new file mode 100644 index 0000000..e55c12c --- /dev/null +++ b/src/test/java/biz/nynja/content/file/download/FileDownloadUtil.java @@ -0,0 +1,50 @@ +package biz.nynja.content.file.download; + +import java.util.UUID; + +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; + +import biz.nynja.content.file.metadata.dto.FileMetadata; + +@TestConfiguration +public class FileDownloadUtil { + + public final static String FILE_STORAGE_LOCATION = "src/test/resources/"; + public final static String VALID_UUID = "d397c291-4d37-49dc-8d55-f6b08ab34fb2"; + public final static String VALID_UUID_NOT_FOUND = "d397c291-4d37-49dc-8d55-f6b08ab34fb3"; + public final static String INVALID_UUID = "d397c291-4d37-49dc-8d55-f6b08ab"; + public final static String TEXT_FILE_EXISTING = "0LDQsNCw0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULnR4dA=="; + public final static String TEXT_FILE_NOT_EXISTING = "notExistingFile.txt"; + public final static String PPTX_FILE_EXISTING = "0LDQsNCw0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULmZpbGU="; + + @Bean + public FileMetadata fileMetaDataExisting() { + FileMetadata fileMetaData = new FileMetadata(); + fileMetaData.setKey(UUID.fromString(VALID_UUID)); + fileMetaData.setStorage("LOCAL"); + fileMetaData.setFileUrl(FILE_STORAGE_LOCATION + TEXT_FILE_EXISTING); + fileMetaData.setFileName(TEXT_FILE_EXISTING); + return fileMetaData; + } + + @Bean + public FileMetadata fileMetaDataNotFound() { + FileMetadata fileMetaData = new FileMetadata(); + fileMetaData.setKey(UUID.fromString(VALID_UUID_NOT_FOUND)); + fileMetaData.setStorage("LOCAL"); + fileMetaData.setFileUrl(FILE_STORAGE_LOCATION + TEXT_FILE_NOT_EXISTING); + fileMetaData.setFileName(TEXT_FILE_NOT_EXISTING); + return fileMetaData; + } + + @Bean + public FileMetadata fileMetaDataOctetStream() { + FileMetadata fileMetaData = new FileMetadata(); + fileMetaData.setKey(UUID.fromString(VALID_UUID)); + fileMetaData.setStorage("LOCAL"); + fileMetaData.setFileUrl(FILE_STORAGE_LOCATION + PPTX_FILE_EXISTING); + fileMetaData.setFileName(PPTX_FILE_EXISTING); + return fileMetaData; + } +} diff --git a/src/test/resources/ONLY_TEST.file b/src/test/resources/0LDQsNCw0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULmZpbGU= similarity index 100% rename from src/test/resources/ONLY_TEST.file rename to src/test/resources/0LDQsNCw0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULmZpbGU= diff --git a/src/test/resources/ONLY_TEST.txt b/src/test/resources/0LDQsNCw0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULnR4dA== similarity index 100% rename from src/test/resources/ONLY_TEST.txt rename to src/test/resources/0LDQsNCw0LDQsNCw0LDQsNCw0LBfT05MWV9URVNULnR4dA== -- GitLab From d3240f8cf9bc8f3a2cc42cab9fc4570bad53103e Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 20 Feb 2019 15:04:49 +0200 Subject: [PATCH 49/70] NY-6949 Create signed Urls for file download - add information about google credentials in readme file; Signed-off-by: abotev-intracol --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 3bfe988..c35b4e8 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,22 @@ The build tool is Maven and there's support for Docker (see Dockerfile), Jenkins The proto files are NOT part of this repo but rather part of the protobuf repo. There's a flow setup in the protobuf repo which scans for .proto file definitions, builds the stub Java files and pushes the generated code/artifacts to an artifactory repo. The generated artifacts are then referenced from the .proto repo as Maven dependency. +To use Google Storage Provider you must set your Google credentials. +To do this First you must have google credentials file which looks like: +{ + "type": "...", + "project_id": "...", + "private_key_id": "...", + "private_key": "-----BEGIN PRIVATE KEY-----\n.....\n-----END PRIVATE KEY-----\n", + "client_email": "...", + "client_id": "...", + "auth_uri": "...", + "token_uri": "...", + "auth_provider_x509_cert_url": "...", + "client_x509_cert_url": "..." +} +Second you must add environment variable on your system with key: GOOGLE_APPLICATION_CREDENTIALS and value: PATH/TO/YOUR_GOOGLE_CREDENTIALS_FILE + # Project Structure ``` -- GitLab From 419dd1e133547d6d5fee718f53f94309ae1f3ed2 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 22 Feb 2019 10:10:04 +0200 Subject: [PATCH 50/70] NY-6949 Create signed Urls for file download - add copyright and fix bug; Signed-off-by: abotev-intracol --- .../biz/nynja/content/file/download/DownloadResponse.java | 8 ++++++++ .../content/file/download/FileDownloadControllerTest.java | 3 --- .../biz/nynja/content/file/download/FileDownloadUtil.java | 8 ++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/download/DownloadResponse.java b/src/main/java/biz/nynja/content/file/download/DownloadResponse.java index 9b3f5ed..d676b4e 100644 --- a/src/main/java/biz/nynja/content/file/download/DownloadResponse.java +++ b/src/main/java/biz/nynja/content/file/download/DownloadResponse.java @@ -1,9 +1,17 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ package biz.nynja.content.file.download; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.core.io.Resource; +/** + * @author Angel.Botev + * + */ + public class DownloadResponse { private HttpStatus httpStatus; diff --git a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java index 6e060c9..e08318a 100644 --- a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java +++ b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java @@ -85,9 +85,6 @@ public class FileDownloadControllerTest { @Autowired private MockMvc mockMvc; - @MockBean - private UploadTokenService uploadTokenService; - @Test public void testFileDownloadRestOK() throws Exception { diff --git a/src/test/java/biz/nynja/content/file/download/FileDownloadUtil.java b/src/test/java/biz/nynja/content/file/download/FileDownloadUtil.java index e55c12c..2ab79b7 100644 --- a/src/test/java/biz/nynja/content/file/download/FileDownloadUtil.java +++ b/src/test/java/biz/nynja/content/file/download/FileDownloadUtil.java @@ -1,3 +1,6 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ package biz.nynja.content.file.download; import java.util.UUID; @@ -7,6 +10,11 @@ import org.springframework.context.annotation.Bean; import biz.nynja.content.file.metadata.dto.FileMetadata; +/** + * @author Angel.Botev + * + */ + @TestConfiguration public class FileDownloadUtil { -- GitLab From cde00d7be9cedff0b21f155dff497244a2740cc1 Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Fri, 22 Feb 2019 10:58:17 +0200 Subject: [PATCH 51/70] Add cronjob and additional env variables. --- charts/content-service/templates/cronjob.yaml | 28 +++++++++++++++++++ .../content-service/templates/deployment.yaml | 16 ++++++++--- releases/dev/content-service.yaml | 13 +++++++-- 3 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 charts/content-service/templates/cronjob.yaml diff --git a/charts/content-service/templates/cronjob.yaml b/charts/content-service/templates/cronjob.yaml new file mode 100644 index 0000000..9c7e037 --- /dev/null +++ b/charts/content-service/templates/cronjob.yaml @@ -0,0 +1,28 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ template "service.fullname" . }}-db-cleanup + labels: + app: {{ template "service.name" . }} + chart: {{ template "service.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + schedule: "0 3 * * *" + failedJobsHistoryLimit: 5 + successfulJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - name: {{ template "service.name" . }}-db-cleanup + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["/bin/sh"] + args: + - -c + - curl -v http://{{ template "service.name" . }}.svc.cluster.local:{{ .Values.ports.containerPort.http }}/rest/jobs/old + restartPolicy: OnFailure diff --git a/charts/content-service/templates/deployment.yaml b/charts/content-service/templates/deployment.yaml index 337847d..7faf027 100644 --- a/charts/content-service/templates/deployment.yaml +++ b/charts/content-service/templates/deployment.yaml @@ -60,9 +60,14 @@ spec: value: {{ .Values.ports.containerPort.grpc | quote }} {{ if .Values.extra_vars -}} {{ toYaml .Values.extra_vars | indent 8 }} -{{- end }} +{{- end -}} + - name: SIGN_URL_KEY + valueFrom: + secretKeyRef: + name: content-service-gcloud + key: sign_url_key volumeMounts: - - name: service-account + - name: service-account-gcloud mountPath: /opt/nynja/config resources: {{ toYaml .Values.resources | indent 12 }} @@ -80,6 +85,9 @@ spec: {{- end }} volumes: - - name: service-account + - name: service-account-gcloud secret: - secretName: service-account + secretName: content-service-gcloud + items: + - key: application-credentials.json + path: application-credentials.json diff --git a/releases/dev/content-service.yaml b/releases/dev/content-service.yaml index 745dc30..71afe05 100644 --- a/releases/dev/content-service.yaml +++ b/releases/dev/content-service.yaml @@ -55,14 +55,21 @@ spec: extra_vars: - name: FILE_UPLOAD_URL - value: https://content.dev-eu.nynja.net/file/upload + value: https://content.dev-eu.nynja.net/file/upload/ - name: FILE_DOWNLOAD_URL value: https://content.dev-eu.nynja.net/file/download/ - name: LOCAL_STORAGE_LOCATION - value: /src/main/resources + value: /opt/nynja/ - name: GOOGLE_STORAGE_URI value: https://storage.googleapis.com - name: GOOGLE_STORAGE_BUCKET - value: content-service-dev + value: content-service-dev-test - name: GOOGLE_APPLICATION_CREDENTIALS value: /opt/nynja/config/application-credentials.json + - name: SIGN_URL_KEY_NAME + value: test2 + - name: SIGN_URL_TTL + value: 86400 + - name: SIGN_URL_CDN_URI + value: http://35.244.165.21 + -- GitLab From 499f3c847934f2eb01264c0d95b45a4030b9c44d Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 22 Feb 2019 11:16:15 +0200 Subject: [PATCH 52/70] NY-6945 [BE] content service > Exception when create a request without Access token - fix bug; Signed-off-by: abotev-intracol --- .../interceptors/ContentServiceInterceptor.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java b/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java index c0f0bfb..66c1ea8 100644 --- a/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java +++ b/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java @@ -44,11 +44,13 @@ public class ContentServiceInterceptor implements ServerInterceptor { String accessToken = getAccessToken(headers.get(ContentServiceConstants.ACCESS_TOKEN_METADATA)); Context ctx = Context.current(); if (!StringUtils.isEmpty(accessToken)) { - ctx = Context.current().withValue(ContentServiceConstants.ACCOUNT_ID, - retrieveAccountId(accessToken)); - } else + ctx = Context.current().withValue(ContentServiceConstants.ACCOUNT_ID, retrieveAccountId(accessToken)); + return Contexts.interceptCall(ctx, call, headers, next); + } else { call.close(Status.PERMISSION_DENIED, new Metadata()); - return Contexts.interceptCall(ctx, call, headers, next); + return new ServerCall.Listener() { + }; + } } private String getAccessToken(String authHeader) { @@ -57,10 +59,15 @@ public class ContentServiceInterceptor implements ServerInterceptor { try { accessToken = authHeader.split(" ")[1]; } catch (Exception e) { - logger.error("Unexpected Authorization Header format: {}", authHeader); + if (authHeader == null) { + logger.error("Missing Authorization Header: {}", authHeader); + } else { + logger.error("Unexpected Authorization Header format: {}", authHeader); + } } return accessToken; } + private String retrieveAccountId(String accessToken) { DecodedJWT decodedToken = JWT.decode(accessToken); String accountIdEncoded = decodedToken.getSubject(); -- GitLab From 801fab1c481b02db61335ed31ad1c6d89a7b7bca Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 26 Feb 2019 14:52:16 +0200 Subject: [PATCH 53/70] NY-7050 Google storage provider issues - upgrade google storage sdk version and fix some issues; Signed-off-by: abotev-intracol --- pom.xml | 3 ++- .../storage/impl/GoogleStorageProvider.java | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index bc4e4ea..6023347 100644 --- a/pom.xml +++ b/pom.xml @@ -165,8 +165,9 @@ com.google.cloud google-cloud-storage - 1.53.0 + 1.63.0 + diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index a687ec2..97e20df 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -7,15 +7,16 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; +import java.net.URLEncoder; import java.nio.file.Files; import java.nio.file.Paths; import java.security.GeneralSecurityException; import java.security.InvalidKeyException; import java.security.Key; import java.security.NoSuchAlgorithmException; +import java.util.Arrays; import java.util.Base64; import java.util.Calendar; -import java.util.Collections; import java.util.Date; import javax.crypto.Mac; @@ -46,6 +47,7 @@ import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageConfiguration; import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.upload.token.UploadTokenService; import io.netty.buffer.ByteBufInputStream; import io.netty.buffer.Unpooled; @@ -63,15 +65,17 @@ public class GoogleStorageProvider implements StorageProvider { private HttpRequestFactory requestFactory; private StorageConfiguration storageConfiguration; + private UploadTokenService uploadTokenService; - public GoogleStorageProvider(StorageConfiguration storageConfiguration) { + public GoogleStorageProvider(StorageConfiguration storageConfiguration, UploadTokenService uploadTokenService) { this.storageConfiguration = storageConfiguration; + this.uploadTokenService = uploadTokenService; HttpTransport httpTransport; try { httpTransport = GoogleNetHttpTransport.newTrustedTransport(); // Build an account credential. GoogleCredential credential = GoogleCredential.getApplicationDefault(); - credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)); + credential = credential.createScoped(Arrays.asList(StorageScopes.DEVSTORAGE_FULL_CONTROL)); requestFactory = httpTransport.createRequestFactory(credential); } catch (GeneralSecurityException | IOException e) { logger.error("Error with Google credentials: {}", e.getMessage()); @@ -82,8 +86,8 @@ public class GoogleStorageProvider implements StorageProvider { @Override public String initialize(String fileName) throws Exception { logger.debug("Initiate resumable Google upload for file: {}", fileName); - String URI = storageConfiguration.getGoogleStorageURI() + "/" + storageConfiguration.getGoogleBucketName() + "/" - + fileName; + String URI = storageConfiguration.getGoogleStorageURI() + "/" + + URLEncoder.encode(storageConfiguration.getGoogleBucketName(), "UTF-8") + "/" + fileName; HttpResponse resp = buildAndExecuteInitializeRequest(URI, fileName); if (resp.getStatusCode() == 201) { String fileLocation = resp.getHeaders().getLocation(); @@ -146,7 +150,7 @@ public class GoogleStorageProvider implements StorageProvider { HttpHeaders headers = new HttpHeaders(); headers.set("x-goog-resumable", "start"); headers.setContentLength((long) 0); - headers.setContentType(Files.probeContentType(Paths.get(fileName))); + headers.setContentType(Files.probeContentType(Paths.get(uploadTokenService.decodeFileName(fileName)))); req.setHeaders(headers); req.setResponseHeaders(headers); logger.debug("Executing POST request to: {}", URI); @@ -196,8 +200,7 @@ public class GoogleStorageProvider implements StorageProvider { try { url = storageConfiguration.getCdnURI() + "/" + fileMetaData.getFileName(); byte[] keyBytes = Base64.getUrlDecoder().decode(storageConfiguration.getSignKey()); - String signedUrl = signUrl(url, keyBytes, - storageConfiguration.getSignKeyName(), cal.getTime()); + String signedUrl = signUrl(url, keyBytes, storageConfiguration.getSignKeyName(), cal.getTime()); signedUrlURI = new URI(signedUrl); } catch (InvalidKeyException | NoSuchAlgorithmException | URISyntaxException e) { -- GitLab From 2c50a2ee4dca9097d984d37731694ede2247c802 Mon Sep 17 00:00:00 2001 From: Angel Botev Date: Thu, 28 Feb 2019 14:31:02 +0200 Subject: [PATCH 54/70] NY-7050 Google storage provider issues - change filename encoding and decoding before uploading; Signed-off-by: Angel Botev --- .../file/storage/impl/GoogleStorageProvider.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 97e20df..07923c2 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -6,7 +6,6 @@ package biz.nynja.content.file.storage.impl; import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.file.Files; import java.nio.file.Paths; @@ -86,9 +85,11 @@ public class GoogleStorageProvider implements StorageProvider { @Override public String initialize(String fileName) throws Exception { logger.debug("Initiate resumable Google upload for file: {}", fileName); + String originalFileName = uploadTokenService.decodeFileName(fileName).substring(11); String URI = storageConfiguration.getGoogleStorageURI() + "/" - + URLEncoder.encode(storageConfiguration.getGoogleBucketName(), "UTF-8") + "/" + fileName; - HttpResponse resp = buildAndExecuteInitializeRequest(URI, fileName); + + URLEncoder.encode(storageConfiguration.getGoogleBucketName(), "UTF-8") + "/" + + URLEncoder.encode(originalFileName, "UTF-8"); + HttpResponse resp = buildAndExecuteInitializeRequest(URI, originalFileName); if (resp.getStatusCode() == 201) { String fileLocation = resp.getHeaders().getLocation(); logger.debug("Successfull Google upload initiation for file: {}, {}", fileName, fileLocation); @@ -150,7 +151,7 @@ public class GoogleStorageProvider implements StorageProvider { HttpHeaders headers = new HttpHeaders(); headers.set("x-goog-resumable", "start"); headers.setContentLength((long) 0); - headers.setContentType(Files.probeContentType(Paths.get(uploadTokenService.decodeFileName(fileName)))); + headers.setContentType(Files.probeContentType(Paths.get(fileName))); req.setHeaders(headers); req.setResponseHeaders(headers); logger.debug("Executing POST request to: {}", URI); @@ -198,12 +199,14 @@ public class GoogleStorageProvider implements StorageProvider { URI signedUrlURI = null; String url = null; try { - url = storageConfiguration.getCdnURI() + "/" + fileMetaData.getFileName(); + String originalFileName = uploadTokenService.decodeFileName(fileMetaData.getFileName()).substring(11); + String urlEncodedFileName = URLEncoder.encode(originalFileName, "UTF-8"); + url = storageConfiguration.getCdnURI() + "/" + urlEncodedFileName; byte[] keyBytes = Base64.getUrlDecoder().decode(storageConfiguration.getSignKey()); String signedUrl = signUrl(url, keyBytes, storageConfiguration.getSignKeyName(), cal.getTime()); signedUrlURI = new URI(signedUrl); - } catch (InvalidKeyException | NoSuchAlgorithmException | URISyntaxException e) { + } catch (Exception e) { logger.debug("Error signing url: {}, {}", url, e.getStackTrace()); throw new InternalError("Error downloading file!"); } -- GitLab From 15209d29933b762e753b4b7875e2db397a72847e Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Fri, 1 Mar 2019 16:05:35 +0200 Subject: [PATCH 55/70] Allow google egress traffic via istio . --- charts/content-service/templates/secret.yaml | 11 ++++++++ .../templates/service-entry-google.yaml | 18 +++++++++++++ .../templates/virtual-service-google.yaml | 24 +++++++++++++++++ releases/dev/content-service.yaml | 26 ++++++++++++------- 4 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 charts/content-service/templates/secret.yaml create mode 100644 charts/content-service/templates/service-entry-google.yaml create mode 100644 charts/content-service/templates/virtual-service-google.yaml diff --git a/charts/content-service/templates/secret.yaml b/charts/content-service/templates/secret.yaml new file mode 100644 index 0000000..fae7bd2 --- /dev/null +++ b/charts/content-service/templates/secret.yaml @@ -0,0 +1,11 @@ +{{ if and .Values.sign_url_key .Values.account_service -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: content-service-gcloud + namespace: content +data: + sign_url_key: {{ .Values.sign_url_key | b64enc }} + application-credentials.json: {{ .Values.account_service }} +{{- end -}} diff --git a/charts/content-service/templates/service-entry-google.yaml b/charts/content-service/templates/service-entry-google.yaml new file mode 100644 index 0000000..b140d31 --- /dev/null +++ b/charts/content-service/templates/service-entry-google.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: ServiceEntry +metadata: + name: {{ template "service.name" . }}-google-api + labels: + app: {{ template "service.name" . }} + chart: {{ template "service.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + hosts: +{{ toYaml .Values.google_api_hosts | indent 2 }} + ports: + - number: 443 + name: https + protocol: HTTPS + resolution: DNS + location: MESH_EXTERNAL diff --git a/charts/content-service/templates/virtual-service-google.yaml b/charts/content-service/templates/virtual-service-google.yaml new file mode 100644 index 0000000..78182b6 --- /dev/null +++ b/charts/content-service/templates/virtual-service-google.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: {{ template "service.fullname" . }}-google-api + labels: + app: {{ template "service.name" . }} + chart: {{ template "service.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + hosts: +{{ toYaml .Values.google_api_hosts | indent 2 }} + tls: +{{- range .Values.google_api_hosts }} + - match: + - port: 443 + sni_hosts: + - {{ . }} + route: + - destination: + host: {{ . }} + port: + number: 443 +{{- end }} diff --git a/releases/dev/content-service.yaml b/releases/dev/content-service.yaml index 71afe05..d5416a5 100644 --- a/releases/dev/content-service.yaml +++ b/releases/dev/content-service.yaml @@ -53,23 +53,31 @@ spec: - x-grpc-web maxAge: "600s" + google_api_hosts: + - www.googleapis.com + - accounts.google.com + - oauth2.googleapis.com + - storage.googleapis.com + + # Allow Egress traffic to Google API extra_vars: - name: FILE_UPLOAD_URL - value: https://content.dev-eu.nynja.net/file/upload/ + value: https://content.dev-eu.nynja.net/rest/file/upload/ - name: FILE_DOWNLOAD_URL - value: https://content.dev-eu.nynja.net/file/download/ + value: https://content.dev-eu.nynja.net/rest/file/download/ - name: LOCAL_STORAGE_LOCATION - value: /opt/nynja/ + value: "/opt/nynja/" - name: GOOGLE_STORAGE_URI - value: https://storage.googleapis.com + value: "https://storage.googleapis.com" - name: GOOGLE_STORAGE_BUCKET - value: content-service-dev-test + value: nynja-content-service-dev - name: GOOGLE_APPLICATION_CREDENTIALS value: /opt/nynja/config/application-credentials.json - name: SIGN_URL_KEY_NAME - value: test2 + value: content-dev-sign-key - name: SIGN_URL_TTL - value: 86400 + value: "86400" - name: SIGN_URL_CDN_URI - value: http://35.244.165.21 - + value: "http://content-cdn.dev-eu.nynja.net" + - name: STORAGE_PROVIDER + value: GOOGLE -- GitLab From dc9eb3f3cff4b3f6757aa54f13752ec1aa7ce67d Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Wed, 6 Mar 2019 12:10:53 +0200 Subject: [PATCH 56/70] Parameterizing max_file_size application property --- releases/dev/content-service.yaml | 2 ++ src/main/resources/application-production.yml | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/releases/dev/content-service.yaml b/releases/dev/content-service.yaml index d5416a5..6c0e553 100644 --- a/releases/dev/content-service.yaml +++ b/releases/dev/content-service.yaml @@ -81,3 +81,5 @@ spec: value: "http://content-cdn.dev-eu.nynja.net" - name: STORAGE_PROVIDER value: GOOGLE + - name: MAX_FILE_SIZE + value: "3072" diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 55139b0..781a3cb 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -21,7 +21,7 @@ token: encryptdecrypt: key: someKeyUsedForEncryptionAndDecryption time_to_live: 36000 # measured in seconds - max_file_size: 1500000000 #measured in bytes (B) + max_file_size: ${MAX_FILE_SIZE:1610612736} #measured in bytes (B) (Default: 1.5G) max_upload_retries: 3 media-types: @@ -86,4 +86,3 @@ management: export: prometheus: enabled: true - -- GitLab From 3f8b7e8c8fc29ec4098b40478e20ac645ff1a790 Mon Sep 17 00:00:00 2001 From: Angel Botev Date: Thu, 7 Mar 2019 18:02:26 +0200 Subject: [PATCH 57/70] NY-7050 Google storage provider issues - cancel google upload; Signed-off-by: Angel Botev --- .../storage/impl/GoogleStorageProvider.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 07923c2..f358320 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -125,23 +125,27 @@ public class GoogleStorageProvider implements StorageProvider { } @Override - public void close(String fileLocation) throws IOException { + public void close(String fileLocation) { logger.debug("Cancel resumable Google upload for location: {}", fileLocation); String URI = fileLocation; GenericUrl url = new GenericUrl(URI); HttpRequest req; - req = requestFactory.buildDeleteRequest(url); - HttpHeaders headers = new HttpHeaders(); - headers.setContentLength((long) 0); - req.setHeaders(headers); - logger.debug("Executing DELETE request to: {}", URI); - // Execute request - HttpResponse resp = req.execute(); - if (resp.getStatusCode() == 200) { - logger.debug("Successfull canceled Google upload for location: {}", fileLocation); - } else { - logger.error("Error canceling Google upload. Status code: {}, message: ", resp.getStatusCode(), - resp.getStatusMessage()); + try { + req = requestFactory.buildDeleteRequest(url); + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength((long) 0); + req.setHeaders(headers); + logger.debug("Executing DELETE request to: {}", URI); + // Execute request + HttpResponse resp = req.execute(); + if (resp.getStatusCode() == 200) { + logger.debug("Successfull canceled Google upload for location: {}", fileLocation); + } else { + logger.error("Error canceling Google upload. Status code: {}, message: ", resp.getStatusCode(), + resp.getStatusMessage()); + } + } catch (IOException e) { + logger.warn("Google canceling upload status for location: {}, {}", fileLocation, e.getMessage()); } } -- GitLab From 344b2f102f95d4012b1acf2717eaeea0f9665c37 Mon Sep 17 00:00:00 2001 From: Angel Botev Date: Fri, 8 Mar 2019 11:06:34 +0200 Subject: [PATCH 58/70] NY-7050 Google storage provider issues - cancel google upload; Signed-off-by: Angel Botev --- .../nynja/content/file/upload/rest/FileUploadController.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java index 759fd82..a0bc304 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java @@ -17,6 +17,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import biz.nynja.content.file.upload.models.UploadStatus; import biz.nynja.content.file.upload.models.UploadStore; import biz.nynja.content.file.upload.rest.validation.ChunkUploadValidator; import biz.nynja.content.file.upload.rest.validation.ValidationResult; @@ -94,6 +95,9 @@ public class FileUploadController { "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished.", HttpStatus.BAD_REQUEST); } + if(uploadInfoResult.get().getStatus().equals(UploadStatus.PENDING)) { + return new ResponseEntity<>("You are not allowed to cancel upload. Upload is not started yet.", HttpStatus.NOT_FOUND); + } if(!fileUploadService.cancelUploadProcess(uploadInfoResult.get())) { return new ResponseEntity<>("Error canceling upload job. File not found or not removed.", HttpStatus.INTERNAL_SERVER_ERROR); } -- GitLab From a5da41a53c5be3154fb9db0e4ea8698c3355f109 Mon Sep 17 00:00:00 2001 From: Nicolas Berthet Date: Fri, 26 Apr 2019 12:38:45 +0800 Subject: [PATCH 59/70] Update the dev deployment to use flux --- releases/dev/content-service.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/releases/dev/content-service.yaml b/releases/dev/content-service.yaml index 6c0e553..a74d7c1 100644 --- a/releases/dev/content-service.yaml +++ b/releases/dev/content-service.yaml @@ -1,10 +1,14 @@ +apiVersion: flux.weave.works/v1beta1 kind: HelmRelease metadata: name: content-service namespace: content spec: + releaseName: content-service chart: + repository: https://nynjagroup.jfrog.io/nynjagroup/helm/ name: content-service + version: 0.1.0 values: replicaCount: 1 -- GitLab From ce5be1d7c6561365d6b1289bf3588230bd9d1f6d Mon Sep 17 00:00:00 2001 From: Nicolas Berthet Date: Tue, 18 Jun 2019 17:49:03 +0800 Subject: [PATCH 60/70] Update the staging deployment to use flux --- releases/staging/content-service.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/releases/staging/content-service.yaml b/releases/staging/content-service.yaml index 7bc22b1..5bf2eff 100644 --- a/releases/staging/content-service.yaml +++ b/releases/staging/content-service.yaml @@ -1,10 +1,14 @@ +apiVersion: flux.weave.works/v1beta1 kind: HelmRelease metadata: name: content-service namespace: content spec: + releaseName: content-service chart: + repository: https://nynjagroup.jfrog.io/nynjagroup/helm/ name: content-service + version: 0.1.0 values: replicaCount: 2 -- GitLab From a95127ff2ce1b018e48d1f893053d3cbc6c3c058 Mon Sep 17 00:00:00 2001 From: Nicolas Berthet Date: Tue, 25 Jun 2019 11:53:20 +0800 Subject: [PATCH 61/70] Update the prod release to use flux --- releases/prod/content-service.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/releases/prod/content-service.yaml b/releases/prod/content-service.yaml index 6271881..1014f38 100644 --- a/releases/prod/content-service.yaml +++ b/releases/prod/content-service.yaml @@ -1,10 +1,14 @@ +apiVersion: flux.weave.works/v1beta1 kind: HelmRelease metadata: name: content-service namespace: content spec: + releaseName: content-service chart: + repository: https://nynjagroup.jfrog.io/nynjagroup/helm/ name: content-service + version: 0.1.0 values: replicaCount: 2 -- GitLab From 6458d29cba9fcd7b16d3f9b1c609ea94f5721c0d Mon Sep 17 00:00:00 2001 From: Nicolas Berthet Date: Thu, 1 Aug 2019 11:49:58 +0800 Subject: [PATCH 62/70] Update the dev dns --- releases/dev/content-service.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/releases/dev/content-service.yaml b/releases/dev/content-service.yaml index a74d7c1..ef76e40 100644 --- a/releases/dev/content-service.yaml +++ b/releases/dev/content-service.yaml @@ -20,7 +20,7 @@ spec: selector: - api-gateway.default.svc.cluster.local hosts: - - content.dev-eu.nynja.net + - content.dev.nynja.net resources: limits: @@ -44,7 +44,7 @@ spec: - http://10.191.224.180:3000 - https://localhost:8080 - https://127.0.0.1:8080 - - https://web.dev-eu.nynja.net + - https://web.dev.nynja.net - https://web.staging.nynja.net - https://web.nynja.net allowMethods: @@ -66,9 +66,9 @@ spec: # Allow Egress traffic to Google API extra_vars: - name: FILE_UPLOAD_URL - value: https://content.dev-eu.nynja.net/rest/file/upload/ + value: https://content.dev.nynja.net/rest/file/upload/ - name: FILE_DOWNLOAD_URL - value: https://content.dev-eu.nynja.net/rest/file/download/ + value: https://content.dev.nynja.net/rest/file/download/ - name: LOCAL_STORAGE_LOCATION value: "/opt/nynja/" - name: GOOGLE_STORAGE_URI @@ -82,7 +82,7 @@ spec: - name: SIGN_URL_TTL value: "86400" - name: SIGN_URL_CDN_URI - value: "http://content-cdn.dev-eu.nynja.net" + value: "http://content-cdn.dev.nynja.net" - name: STORAGE_PROVIDER value: GOOGLE - name: MAX_FILE_SIZE -- GitLab From bb6fb1629b62fae3cf0f6675bee0e51208ba1375 Mon Sep 17 00:00:00 2001 From: Angel Botev Date: Thu, 1 Aug 2019 14:40:18 +0300 Subject: [PATCH 63/70] Add fix for service shutdown Signed-off-by: Angel Botev --- .../content/healthindicators/GrpcServerHealthIndicator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/biz/nynja/content/healthindicators/GrpcServerHealthIndicator.java b/src/main/java/biz/nynja/content/healthindicators/GrpcServerHealthIndicator.java index 433bedc..accad54 100644 --- a/src/main/java/biz/nynja/content/healthindicators/GrpcServerHealthIndicator.java +++ b/src/main/java/biz/nynja/content/healthindicators/GrpcServerHealthIndicator.java @@ -47,6 +47,10 @@ public class GrpcServerHealthIndicator implements HealthIndicator { ContentServiceGrpc.getServiceDescriptor().getName(), e.getMessage()); LOGGER.debug(null, e.getCause()); return Health.down().build(); + } finally { + if (channel != null) { + channel.shutdown(); + } } switch (servingStatus) { -- GitLab From 3c44ac68fdd33b8f5bfaca60ce484a7777f36a75 Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Tue, 10 Sep 2019 18:08:16 +0300 Subject: [PATCH 64/70] Enable jwt authentication and update of the cdn url. --- .../templates/authentication-policy.yaml | 41 ++++++++++--------- releases/dev/content-service.yaml | 2 +- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/charts/content-service/templates/authentication-policy.yaml b/charts/content-service/templates/authentication-policy.yaml index 895af58..be24b27 100644 --- a/charts/content-service/templates/authentication-policy.yaml +++ b/charts/content-service/templates/authentication-policy.yaml @@ -1,19 +1,22 @@ -#apiVersion: "authentication.istio.io/v1alpha1" -#kind: "Policy" -#metadata: -# name: {{ template "service.name" . }} -# labels: -# app: {{ template "service.name" . }} -# chart: {{ template "service.chart" . }} -# release: {{ .Release.Name }} -# heritage: {{ .Release.Service }} -#spec: -# targets: -# - name: {{ template "service.name" . }} -# origins: -# - jwt: -# issuer: https://auth.nynja.biz/ -# jwksUri: http://auth-service.auth.svc.cluster.local:8008/keys/public -# audiences: -# - dGVzdEluc3RhbmNl:NynjaApp:NynjaOrg -# principalBinding: USE_ORIGIN +apiVersion: "authentication.istio.io/v1alpha1" +kind: "Policy" +metadata: + name: {{ template "service.name" . }} + labels: + app: {{ template "service.name" . }} + chart: {{ template "service.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + targets: + - name: {{ template "service.name" . }} + origins: + - jwt: + issuer: https://auth.nynja.biz/ + jwksUri: http://auth-service.auth.svc.cluster.local:8008/keys/public + trigger_rules: + - excluded_paths: + - exact: /actuator/health + - exact: /actuator/info + - exact: /actuator/status + principalBinding: USE_ORIGIN diff --git a/releases/dev/content-service.yaml b/releases/dev/content-service.yaml index ef76e40..8f69cb8 100644 --- a/releases/dev/content-service.yaml +++ b/releases/dev/content-service.yaml @@ -82,7 +82,7 @@ spec: - name: SIGN_URL_TTL value: "86400" - name: SIGN_URL_CDN_URI - value: "http://content-cdn.dev.nynja.net" + value: "http://content-cdn.dev-eu.nynja.net" - name: STORAGE_PROVIDER value: GOOGLE - name: MAX_FILE_SIZE -- GitLab From 52d2d1fd2bce380b6c10250c749dddaa000294f4 Mon Sep 17 00:00:00 2001 From: Stoyan Tzenkov Date: Thu, 12 Sep 2019 11:06:45 +0300 Subject: [PATCH 65/70] FILE_VERSIONING: Got rid of constructed filename. Signed-off-by: Stoyan Tzenkov --- .../file/storage/impl/GoogleStorageProvider.java | 4 ++-- .../file/storage/impl/LocalStorageProvider.java | 2 -- .../content/file/upload/rest/FileUploadService.java | 8 +------- src/main/resources/application-dev.yml | 8 ++++---- src/main/resources/content_service_gcs_dev.json | 12 ++++++++++++ 5 files changed, 19 insertions(+), 15 deletions(-) create mode 100644 src/main/resources/content_service_gcs_dev.json diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index f358320..3efb9dc 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -85,7 +85,7 @@ public class GoogleStorageProvider implements StorageProvider { @Override public String initialize(String fileName) throws Exception { logger.debug("Initiate resumable Google upload for file: {}", fileName); - String originalFileName = uploadTokenService.decodeFileName(fileName).substring(11); + String originalFileName = uploadTokenService.decodeFileName(fileName); String URI = storageConfiguration.getGoogleStorageURI() + "/" + URLEncoder.encode(storageConfiguration.getGoogleBucketName(), "UTF-8") + "/" + URLEncoder.encode(originalFileName, "UTF-8"); @@ -203,7 +203,7 @@ public class GoogleStorageProvider implements StorageProvider { URI signedUrlURI = null; String url = null; try { - String originalFileName = uploadTokenService.decodeFileName(fileMetaData.getFileName()).substring(11); + String originalFileName = uploadTokenService.decodeFileName(fileMetaData.getFileName()); String urlEncodedFileName = URLEncoder.encode(originalFileName, "UTF-8"); url = storageConfiguration.getCdnURI() + "/" + urlEncodedFileName; byte[] keyBytes = Base64.getUrlDecoder().decode(storageConfiguration.getSignKey()); diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index ba89c70..79028f0 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -115,8 +115,6 @@ public class LocalStorageProvider implements StorageProvider { } if (fileName == null) { fileName = RandomStringUtils.randomAlphabetic(15); - } else { - fileName = fileName.substring(11); } HttpHeaders httpHeaders = new HttpHeaders(); diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index 39b046f..2c4e435 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -83,8 +83,7 @@ public class FileUploadService { private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { String fileName = uploadTokenService.decodeFileName(uploadInfo.getFileName()); - String constructedFilename = constructFileName(fileName); - String encodedFileName = uploadTokenService.encodeFileName(constructedFilename); + String encodedFileName = uploadTokenService.encodeFileName(fileName); uploadInfo.setFileName(encodedFileName); String uploadLocation = storageProvider.initialize(encodedFileName); @@ -103,11 +102,6 @@ public class FileUploadService { return dowlnoadUrl; } - private String constructFileName(String fileName) { - String generatedPrefix = RandomStringUtils.randomAlphabetic(10); - return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); - } - private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { uploadInfo.setPartId(partId); uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 945a5bf..f6c0548 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -48,13 +48,13 @@ storage: location: src/main/resources google: uri: https://storage.googleapis.com - bucket: content-service-dev + bucket: nynja-content-service-dev upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) sign_url: - key_name: content-service-dev-key - key: dsankldmsakdmkalsdmksa== + key_name: content-dev-sign-key + key: AyeUSgO40k3EqFxrXFWSOA== ttl: 86400 # measured in seconds. - cdn_uri: http://35.244.165.21 + cdn_uri: http://content-cdn.dev-eu.nynja.net # To enable colors in Eclipse: diff --git a/src/main/resources/content_service_gcs_dev.json b/src/main/resources/content_service_gcs_dev.json new file mode 100644 index 0000000..2886dda --- /dev/null +++ b/src/main/resources/content_service_gcs_dev.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "nynja-ci-201610", + "private_key_id": "a79ffba070f62c754953f02019b16713e7e6b973", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQD20QxZt9dfAcKj\nnPJ8YMkpcQnjsmHGyYUzsFB81bxMQH1ls+cXS69cA5rlNk908KEDpYYFE7K0a7v0\nRVaTN6tBitwFa/nsklN0VPLiCOBTx3G9tL0hab65p8mAWUmBFHf6M08Z1PWJBYgt\nGY1oXUt+Re6bns76JKAGBmv9Av8lu4jztfE4fTulXHvv70du0+RxWACxzbOS/tIm\nTCQXw0iFd9v7m56gYoV0vCujwOY9x4b6uwzPaUGf3zlh+EqofjNnf01p25GdtMsM\n6/rZNS2Sk17+V4TEdV6QThsnhBuX9TEs/yZxstxIZJLiyDKQ8zI0CRWslcYMOpqH\ngoVZEK5FAgMBAAECggEACysTcX3Zn4FvYsLn0Av7HkZwqZ55QSr+PYQFaFitkA+8\n/NyioNG6aiH5oNSPzN1Uk9sqENG+3N4hH6EBlWDH/YvAf/hOU0+duh5b7TF0oSXd\ntjVW94uW50S53/ZqVo5sDWTD+AnEYIJhEpVc6jOKhni4ns8VU4lTumIRAd5gWD8o\n6COgn+TGeCu4g9w/dfY/zCFgh3qV9KNBxG9gma6Au8E+vipcqgs0B87K53sDDQje\nFnIIJkcvD49rTNudZSttKkRK4i9mQpW0ocQwnzL/2oVsvaa8NFZ5NPMWYlerP1oB\n/26r6TKL0/t7ezTzYvWtHCXsin5GfCP8QnwIZfSPIwKBgQD7nKJ9DBVqxpc5JLjT\n6gb1u1WZVkmjThGmJK0XcoExuQXQkTni60iisAMEdcnpu3F+gfZj/FKLNtT2DAcj\n7TKjh9+E/fQ4/JlrtmyFqQZStEKXP7ZneMoD8a5RbZCaLzwRcsx0R5mZ1qOZGRI+\nyVj5wME4HvkFxt8h78wegvGKzwKBgQD7HwEWNKiHdFtzxt+qcnsz001SLUdAor6b\nb83Vx7yMyLw995opeqt35VOdglAHbOQdgh/Pxid5RTdscjibgZoA5p7G9LTDi6hp\nGFvFhjJNriorfOTkkrNKyRI6QitOu+BUOADxR7ZT4TeXRgazxtHwH8zrt8Fa7m30\nFa4f4J0qqwKBgQC4fJfzokmN2gEFqDRRRXZG9U4FOWyS7eoo7RlxaMbr42MtE17j\nUGsYxn/f5FHhLExCWx6DH36SjujFY/pUUFZh631rsR1CL4x6mu6lblKxj3SIG/It\nmjZdf/a1w+VT+DBOBuN8mwc38lZQKj7jhJTVw4ypDqS+pJv08TO0Z4SdpwKBgQCL\nagsQaOunR9H91MsXtbbwbr7egISuPavUX53Gsbzb7BWV0YCkxpWrWoWxTEdh1/JM\n75D5qOkBSowaiM8khi8ZkO9VRUWVylMVBZDteDM1LRv8SLpKq6cePcMA1Pt2GpfQ\nKcdcLJPsvXw/X4EZDag/x2fc9YdDFFu2xtKEoA6hQQKBgQC0OB7lsVhMtuZYpS2V\nG6x0zcL5HEs8nf8EZhPCROnbGkb/U619ybs5uN0D2/dpUEMuJgf/4JV+wxF22Dbu\n464JWP+iucnUIWRx7BrhVkT70Ya4QArwUw8ItFb4oYMeUarbp12pK2bwMi5OXe8/\nPGhAtGYo9LQpeDk/+ka8vN9MXA==\n-----END PRIVATE KEY-----\n", + "client_email": "content-service-dev@nynja-ci-201610.iam.gserviceaccount.com", + "client_id": "103259379397107638453", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/content-service-dev%40nynja-ci-201610.iam.gserviceaccount.com" +} -- GitLab From d67e332eafa9ff4c0d7b7facf45e1237a1edc9b7 Mon Sep 17 00:00:00 2001 From: Stoyan Tzenkov Date: Thu, 12 Sep 2019 13:56:53 +0300 Subject: [PATCH 66/70] FILE_VERSIONING: Organized uploaded files in folders by account ID. Signed-off-by: Stoyan Tzenkov --- .../content/file/storage/StorageProvider.java | 5 +++-- .../file/storage/impl/GoogleStorageProvider.java | 14 ++++++++++---- .../file/storage/impl/LocalStorageProvider.java | 13 +++++++------ .../file/upload/rest/FileUploadService.java | 4 ++-- .../content/grpc/services/ContentServiceImpl.java | 4 ++-- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index a57c6ff..9c09fe8 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -5,6 +5,7 @@ package biz.nynja.content.file.storage; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.UUID; import javax.servlet.http.HttpServletRequest; @@ -17,13 +18,13 @@ import biz.nynja.content.file.metadata.dto.FileMetadata; */ public interface StorageProvider { - public String initialize(String fileName) throws Exception; + public String initialize(String fileName, UUID accountId) throws Exception; public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk, int lastUploadedByte) throws Exception; public void close(String fileLocation) throws IOException; - public String getFileUrl(String filename); + public String getFileUrl(String filename, UUID accountId); public Provider getProviderType(); diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 3efb9dc..9713443 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -17,6 +17,7 @@ import java.util.Arrays; import java.util.Base64; import java.util.Calendar; import java.util.Date; +import java.util.UUID; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -83,11 +84,12 @@ public class GoogleStorageProvider implements StorageProvider { } @Override - public String initialize(String fileName) throws Exception { + public String initialize(String fileName, UUID accountId) throws Exception { logger.debug("Initiate resumable Google upload for file: {}", fileName); String originalFileName = uploadTokenService.decodeFileName(fileName); String URI = storageConfiguration.getGoogleStorageURI() + "/" + URLEncoder.encode(storageConfiguration.getGoogleBucketName(), "UTF-8") + "/" + + URLEncoder.encode(accountId.toString(), "UTF-8") + "/" + URLEncoder.encode(originalFileName, "UTF-8"); HttpResponse resp = buildAndExecuteInitializeRequest(URI, originalFileName); if (resp.getStatusCode() == 201) { @@ -183,10 +185,12 @@ public class GoogleStorageProvider implements StorageProvider { } @Override - public String getFileUrl(String filename) { + public String getFileUrl(String filename, UUID accountId) { StringBuilder builder = new StringBuilder(); builder.append(storageConfiguration.getGoogleStorageURI()).append("/") - .append(storageConfiguration.getGoogleBucketName()).append("/").append(filename); + .append(storageConfiguration.getGoogleBucketName()).append("/") + .append(accountId.toString()).append("/") + .append(filename); return builder.toString(); } @@ -205,7 +209,9 @@ public class GoogleStorageProvider implements StorageProvider { try { String originalFileName = uploadTokenService.decodeFileName(fileMetaData.getFileName()); String urlEncodedFileName = URLEncoder.encode(originalFileName, "UTF-8"); - url = storageConfiguration.getCdnURI() + "/" + urlEncodedFileName; + url = storageConfiguration.getCdnURI() + "/" + + fileMetaData.getAccountId().toString() + "/" + + urlEncodedFileName; byte[] keyBytes = Base64.getUrlDecoder().decode(storageConfiguration.getSignKey()); String signedUrl = signUrl(url, keyBytes, storageConfiguration.getSignKeyName(), cal.getTime()); signedUrlURI = new URI(signedUrl); diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 79028f0..aa60d72 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.net.MalformedURLException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.UUID; import javax.servlet.http.HttpServletRequest; @@ -51,8 +52,8 @@ public class LocalStorageProvider implements StorageProvider { } @Override - public String initialize(String fileName) throws FileNotFoundException { - String filePath = constructFilePath(storageConfiguration.getLocalStorageLocation(), fileName); + public String initialize(String fileName, UUID accountId) throws FileNotFoundException { + String filePath = constructFilePath(storageConfiguration.getLocalStorageLocation(), fileName, accountId); logger.debug("Initialized location {}, for file {}", filePath, fileName); return filePath; } @@ -76,13 +77,13 @@ public class LocalStorageProvider implements StorageProvider { logger.debug("Successfull canceled upload for location: {}", fileLocation); } - private String constructFilePath(String contentStoreLocation, String fileName) { - return new StringBuilder(contentStoreLocation).append(File.separator).append(fileName).toString(); + private String constructFilePath(String contentStoreLocation, String fileName, UUID accountId) { + return new StringBuilder(contentStoreLocation).append(File.separator).append(accountId.toString()).append(File.separator).append(fileName).toString(); } @Override - public String getFileUrl(String filename) { - return constructFilePath(storageConfiguration.getLocalStorageLocation(), filename); + public String getFileUrl(String filename, UUID accountId) { + return constructFilePath(storageConfiguration.getLocalStorageLocation(), filename, accountId); } @Override diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index 2c4e435..80dcf72 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -86,7 +86,7 @@ public class FileUploadService { String encodedFileName = uploadTokenService.encodeFileName(fileName); uploadInfo.setFileName(encodedFileName); - String uploadLocation = storageProvider.initialize(encodedFileName); + String uploadLocation = storageProvider.initialize(encodedFileName, uploadInfo.getAccountId()); uploadInfo.setUploadLocation(uploadLocation); uploadInfo.setStorage(storageProvider.getProviderType().toString()); return uploadInfo; @@ -95,7 +95,7 @@ public class FileUploadService { public String finalizeUpload(UploadStore uploadInfo) { FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, - storageProvider.getFileUrl(uploadInfo.getFileName()), storageProvider.getProviderType()); + storageProvider.getFileUrl(uploadInfo.getFileName(), uploadInfo.getAccountId()), storageProvider.getProviderType()); String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); uploadTokenService.deleteToken(uploadInfo.getUploadToken()); logger.info("File sucessfully uploaded."); diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 885d50c..27d6a89 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -175,7 +175,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas if (storageProvider == null) { storageProvider = storageProviderPool.getStorageProviderByType(provider); fileName = constructFileName(uploadInfo.getFileName()); - fileLocation = storageProvider.initialize(fileName); + fileLocation = storageProvider.initialize(fileName, uploadInfo.getAccountId()); uploadInfo.setStatus(UploadStatus.IN_PROGRESS); if (!uploadTokenService.storeToken(uploadInfo, true)) { handleStreamTermination(); @@ -220,7 +220,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas return; } FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, - storageProvider.getFileUrl(fileName), storageProvider.getProviderType()); + storageProvider.getFileUrl(fileName, uploadInfo.getAccountId()), storageProvider.getProviderType()); responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); -- GitLab From e4e23d1b6a6aab4021718032a2b4553a53931146 Mon Sep 17 00:00:00 2001 From: Stoyan Tzenkov Date: Thu, 12 Sep 2019 18:15:37 +0300 Subject: [PATCH 67/70] FILE_VERSIONING: Timestamping files to provide some versioning. Signed-off-by: Stoyan Tzenkov --- .../storage/impl/GoogleStorageProvider.java | 24 +++++++++++++++++-- .../file/upload/rest/FileUploadService.java | 16 ++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 9713443..63198dc 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -25,6 +25,7 @@ import javax.servlet.http.HttpServletRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.ContentDisposition; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; @@ -206,9 +207,10 @@ public class GoogleStorageProvider implements StorageProvider { cal.add(Calendar.SECOND, storageConfiguration.getSignedUrlTTL()); // minus number would decrement the time URI signedUrlURI = null; String url = null; + String decodedFileName = null; try { - String originalFileName = uploadTokenService.decodeFileName(fileMetaData.getFileName()); - String urlEncodedFileName = URLEncoder.encode(originalFileName, "UTF-8"); + decodedFileName = uploadTokenService.decodeFileName(fileMetaData.getFileName()); + String urlEncodedFileName = URLEncoder.encode(decodedFileName, "UTF-8"); url = storageConfiguration.getCdnURI() + "/" + fileMetaData.getAccountId().toString() + "/" + urlEncodedFileName; @@ -224,9 +226,27 @@ public class GoogleStorageProvider implements StorageProvider { org.springframework.http.HttpHeaders httpHeaders = new org.springframework.http.HttpHeaders(); httpHeaders.setLocation(signedUrlURI); + String originalFileName = getOriginalFileName(decodedFileName); + ContentDisposition contentDisposition = ContentDisposition.builder("attachment").filename(originalFileName).build(); + httpHeaders.setContentDisposition(contentDisposition); + return new DownloadResponse(HttpStatus.PERMANENT_REDIRECT, "text/plain", httpHeaders, null); } + private String getOriginalFileName(String fileName) { + String justName = fileName; + String theRest = ""; + + int period = fileName.indexOf("."); + if ( period != -1 ) { + justName = fileName.substring(0, period); + theRest = fileName.substring(period); + } + + int tire = justName.lastIndexOf("-"); + return justName.substring(0, tire) + theRest; + } + private String signUrl(String url, byte[] key, String keyName, Date expirationTime) throws InvalidKeyException, NoSuchAlgorithmException { diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index 80dcf72..af66cb5 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -5,6 +5,7 @@ package biz.nynja.content.file.upload.rest; import java.io.IOException; import java.time.Instant; +import java.util.Date; import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; @@ -83,7 +84,8 @@ public class FileUploadService { private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { String fileName = uploadTokenService.decodeFileName(uploadInfo.getFileName()); - String encodedFileName = uploadTokenService.encodeFileName(fileName); + String timestampedFilename = timestampFileName(fileName); + String encodedFileName = uploadTokenService.encodeFileName(timestampedFilename); uploadInfo.setFileName(encodedFileName); String uploadLocation = storageProvider.initialize(encodedFileName, uploadInfo.getAccountId()); @@ -102,6 +104,18 @@ public class FileUploadService { return dowlnoadUrl; } + private String timestampFileName(String fileName) { + long timestamp = Instant.now().toEpochMilli(); + String justName = fileName; + String theRest = ""; + int period = fileName.indexOf("."); + if ( period != -1 ) { + justName = fileName.substring(0, period); + theRest = fileName.substring(period); + } + return new StringBuilder(justName).append('-').append(Long.toString(timestamp)).append(theRest).toString(); + } + private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { uploadInfo.setPartId(partId); uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); -- GitLab From e176645b92512b61b84fb183dcc2c9e0809df6ae Mon Sep 17 00:00:00 2001 From: Stoyan Tzenkov Date: Fri, 13 Sep 2019 11:56:43 +0300 Subject: [PATCH 68/70] FILE_VERSIONING: Placed the file timestamp in front. Signed-off-by: Stoyan Tzenkov --- .../storage/impl/GoogleStorageProvider.java | 14 +++++-------- .../storage/impl/LocalStorageProvider.java | 21 ++++++++++++++----- .../file/upload/rest/FileUploadService.java | 9 +------- src/main/resources/application-dev.yml | 4 ++-- .../resources/content_service_gcs_dev.json | 12 ----------- 5 files changed, 24 insertions(+), 36 deletions(-) delete mode 100644 src/main/resources/content_service_gcs_dev.json diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 63198dc..d50966d 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -234,17 +234,13 @@ public class GoogleStorageProvider implements StorageProvider { } private String getOriginalFileName(String fileName) { - String justName = fileName; - String theRest = ""; + String originalFileName = fileName; - int period = fileName.indexOf("."); - if ( period != -1 ) { - justName = fileName.substring(0, period); - theRest = fileName.substring(period); + int underscore = fileName.indexOf("_"); + if ( underscore != -1 ) { + originalFileName = fileName.substring(underscore+1); } - - int tire = justName.lastIndexOf("-"); - return justName.substring(0, tire) + theRest; + return originalFileName; } private String signUrl(String url, byte[] key, String keyName, Date expirationTime) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index aa60d72..866c07b 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -102,10 +102,11 @@ public class LocalStorageProvider implements StorageProvider { } // Try to determine file's content type String contentType = null; - String fileName = null; + String fileName, originalFileName = null; try { fileName = uploadTokenService.decodeFileName(resource.getFile().getName()); - contentType = request.getServletContext().getMimeType(fileName); + originalFileName = getOriginalFileName(fileName); + contentType = request.getServletContext().getMimeType(originalFileName); } catch (Exception ex) { logger.info("Could not determine file type."); } @@ -114,16 +115,26 @@ public class LocalStorageProvider implements StorageProvider { if (contentType == null) { contentType = "application/octet-stream"; } - if (fileName == null) { - fileName = RandomStringUtils.randomAlphabetic(15); + if (originalFileName == null) { + originalFileName = RandomStringUtils.randomAlphabetic(15); } HttpHeaders httpHeaders = new HttpHeaders(); - httpHeaders.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileName + "\""); + httpHeaders.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + originalFileName + "\""); return new DownloadResponse(HttpStatus.OK, contentType, httpHeaders, resource); } + private String getOriginalFileName(String fileName) { + String originalFileName = fileName; + + int underscore = fileName.indexOf("_"); + if ( underscore != -1 ) { + originalFileName = fileName.substring(underscore+1); + } + return originalFileName; + } + private Resource getFileContent(FileMetadata fileMetaData) throws FileNotFoundException, MalformedURLException { String fileName = fileMetaData.getFileUrl(); return loadFileAsResource(fileName); diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index af66cb5..f34d4e0 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -106,14 +106,7 @@ public class FileUploadService { private String timestampFileName(String fileName) { long timestamp = Instant.now().toEpochMilli(); - String justName = fileName; - String theRest = ""; - int period = fileName.indexOf("."); - if ( period != -1 ) { - justName = fileName.substring(0, period); - theRest = fileName.substring(period); - } - return new StringBuilder(justName).append('-').append(Long.toString(timestamp)).append(theRest).toString(); + return new StringBuilder(Long.toString(timestamp)).append('_').append(fileName).toString(); } private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index f6c0548..98f2571 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -51,8 +51,8 @@ storage: bucket: nynja-content-service-dev upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) sign_url: - key_name: content-dev-sign-key - key: AyeUSgO40k3EqFxrXFWSOA== + key_name: + key: ttl: 86400 # measured in seconds. cdn_uri: http://content-cdn.dev-eu.nynja.net diff --git a/src/main/resources/content_service_gcs_dev.json b/src/main/resources/content_service_gcs_dev.json deleted file mode 100644 index 2886dda..0000000 --- a/src/main/resources/content_service_gcs_dev.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "service_account", - "project_id": "nynja-ci-201610", - "private_key_id": "a79ffba070f62c754953f02019b16713e7e6b973", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQD20QxZt9dfAcKj\nnPJ8YMkpcQnjsmHGyYUzsFB81bxMQH1ls+cXS69cA5rlNk908KEDpYYFE7K0a7v0\nRVaTN6tBitwFa/nsklN0VPLiCOBTx3G9tL0hab65p8mAWUmBFHf6M08Z1PWJBYgt\nGY1oXUt+Re6bns76JKAGBmv9Av8lu4jztfE4fTulXHvv70du0+RxWACxzbOS/tIm\nTCQXw0iFd9v7m56gYoV0vCujwOY9x4b6uwzPaUGf3zlh+EqofjNnf01p25GdtMsM\n6/rZNS2Sk17+V4TEdV6QThsnhBuX9TEs/yZxstxIZJLiyDKQ8zI0CRWslcYMOpqH\ngoVZEK5FAgMBAAECggEACysTcX3Zn4FvYsLn0Av7HkZwqZ55QSr+PYQFaFitkA+8\n/NyioNG6aiH5oNSPzN1Uk9sqENG+3N4hH6EBlWDH/YvAf/hOU0+duh5b7TF0oSXd\ntjVW94uW50S53/ZqVo5sDWTD+AnEYIJhEpVc6jOKhni4ns8VU4lTumIRAd5gWD8o\n6COgn+TGeCu4g9w/dfY/zCFgh3qV9KNBxG9gma6Au8E+vipcqgs0B87K53sDDQje\nFnIIJkcvD49rTNudZSttKkRK4i9mQpW0ocQwnzL/2oVsvaa8NFZ5NPMWYlerP1oB\n/26r6TKL0/t7ezTzYvWtHCXsin5GfCP8QnwIZfSPIwKBgQD7nKJ9DBVqxpc5JLjT\n6gb1u1WZVkmjThGmJK0XcoExuQXQkTni60iisAMEdcnpu3F+gfZj/FKLNtT2DAcj\n7TKjh9+E/fQ4/JlrtmyFqQZStEKXP7ZneMoD8a5RbZCaLzwRcsx0R5mZ1qOZGRI+\nyVj5wME4HvkFxt8h78wegvGKzwKBgQD7HwEWNKiHdFtzxt+qcnsz001SLUdAor6b\nb83Vx7yMyLw995opeqt35VOdglAHbOQdgh/Pxid5RTdscjibgZoA5p7G9LTDi6hp\nGFvFhjJNriorfOTkkrNKyRI6QitOu+BUOADxR7ZT4TeXRgazxtHwH8zrt8Fa7m30\nFa4f4J0qqwKBgQC4fJfzokmN2gEFqDRRRXZG9U4FOWyS7eoo7RlxaMbr42MtE17j\nUGsYxn/f5FHhLExCWx6DH36SjujFY/pUUFZh631rsR1CL4x6mu6lblKxj3SIG/It\nmjZdf/a1w+VT+DBOBuN8mwc38lZQKj7jhJTVw4ypDqS+pJv08TO0Z4SdpwKBgQCL\nagsQaOunR9H91MsXtbbwbr7egISuPavUX53Gsbzb7BWV0YCkxpWrWoWxTEdh1/JM\n75D5qOkBSowaiM8khi8ZkO9VRUWVylMVBZDteDM1LRv8SLpKq6cePcMA1Pt2GpfQ\nKcdcLJPsvXw/X4EZDag/x2fc9YdDFFu2xtKEoA6hQQKBgQC0OB7lsVhMtuZYpS2V\nG6x0zcL5HEs8nf8EZhPCROnbGkb/U619ybs5uN0D2/dpUEMuJgf/4JV+wxF22Dbu\n464JWP+iucnUIWRx7BrhVkT70Ya4QArwUw8ItFb4oYMeUarbp12pK2bwMi5OXe8/\nPGhAtGYo9LQpeDk/+ka8vN9MXA==\n-----END PRIVATE KEY-----\n", - "client_email": "content-service-dev@nynja-ci-201610.iam.gserviceaccount.com", - "client_id": "103259379397107638453", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/content-service-dev%40nynja-ci-201610.iam.gserviceaccount.com" -} -- GitLab From 2306e8f06a7453f1e033622b99a1b0a9bd35279b Mon Sep 17 00:00:00 2001 From: Stoyan Tzenkov Date: Mon, 16 Sep 2019 13:31:39 +0300 Subject: [PATCH 69/70] FIX_LOCAL_STORAGE: Folder created if does not exist for the account. Signed-off-by: Stoyan Tzenkov --- .../file/storage/impl/LocalStorageProvider.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 866c07b..1da2fef 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -53,7 +53,12 @@ public class LocalStorageProvider implements StorageProvider { @Override public String initialize(String fileName, UUID accountId) throws FileNotFoundException { - String filePath = constructFilePath(storageConfiguration.getLocalStorageLocation(), fileName, accountId); + String originalFileName = uploadTokenService.decodeFileName(fileName); + String folderPath = getFolderPath(storageConfiguration.getLocalStorageLocation(), accountId); + if (!Files.exists(Paths.get(folderPath))) { + new File(folderPath).mkdir(); + } + String filePath = constructFilePath(storageConfiguration.getLocalStorageLocation(), originalFileName, accountId); logger.debug("Initialized location {}, for file {}", filePath, fileName); return filePath; } @@ -77,6 +82,10 @@ public class LocalStorageProvider implements StorageProvider { logger.debug("Successfull canceled upload for location: {}", fileLocation); } + private String getFolderPath(String contentStoreLocation, UUID accountId) { + return new StringBuilder(contentStoreLocation).append(File.separator).append(accountId.toString()).toString(); + } + private String constructFilePath(String contentStoreLocation, String fileName, UUID accountId) { return new StringBuilder(contentStoreLocation).append(File.separator).append(accountId.toString()).append(File.separator).append(fileName).toString(); } -- GitLab From 841e6fe4dc028da1ba1df0762cfd7e24ca05a3d8 Mon Sep 17 00:00:00 2001 From: Stoyan Tzenkov Date: Mon, 16 Sep 2019 15:39:13 +0300 Subject: [PATCH 70/70] FIX_LOCAL_STORAGE: Download fixed as well. Signed-off-by: Stoyan Tzenkov --- .../content/file/storage/impl/LocalStorageProvider.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 1da2fef..384185c 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -113,7 +113,7 @@ public class LocalStorageProvider implements StorageProvider { String contentType = null; String fileName, originalFileName = null; try { - fileName = uploadTokenService.decodeFileName(resource.getFile().getName()); + fileName = resource.getFile().getName(); originalFileName = getOriginalFileName(fileName); contentType = request.getServletContext().getMimeType(originalFileName); } catch (Exception ex) { @@ -145,8 +145,11 @@ public class LocalStorageProvider implements StorageProvider { } private Resource getFileContent(FileMetadata fileMetaData) throws FileNotFoundException, MalformedURLException { - String fileName = fileMetaData.getFileUrl(); - return loadFileAsResource(fileName); + String originalFileName = uploadTokenService.decodeFileName(fileMetaData.getFileName()); + String filePath = storageConfiguration.getLocalStorageLocation() + "/" + + fileMetaData.getAccountId().toString() + "/" + + originalFileName; + return loadFileAsResource(filePath); } private Resource loadFileAsResource(String fileName) throws FileNotFoundException, MalformedURLException { -- GitLab