From 3a61c57e714ab281e0438a0e51761a8b3a68795c Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 21 Dec 2018 15:45:07 +0200 Subject: [PATCH 01/29] NY-6318 Update file metadata - add enum providers; Signed-off-by: abotev-intracol --- .../content/file/metadata/FileMetadataService.java | 5 +++-- .../content/file/storage/StorageProviderPool.java | 8 ++++---- .../configuration/ContentServiceConfiguration.java | 13 ++++++++++--- .../content/grpc/services/ContentServiceImpl.java | 6 ++++-- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index 483d9fe..c84e17f 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.metadata.repositories.FileMetadataReposiory; +import biz.nynja.content.file.storage.Providers; import biz.nynja.content.upload.models.UploadStore; /** @@ -29,7 +30,7 @@ public class FileMetadataService { this.fileMetadataRepository = fileMetadataRepository; } - public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl) { + public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl, Providers storageProvider) { FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setKey(UUID.randomUUID()); fileMetadata.setFileName(fileName); @@ -39,7 +40,7 @@ public class FileMetadataService { fileMetadata.setUploadTimestamp(Instant.now().toEpochMilli()); fileMetadata.setAccountId(uploadInfo.getAccountId()); fileMetadata.setDeviceId(uploadInfo.getDeviceId()); - fileMetadata.setStorage("LOCAL"); + fileMetadata.setStorage(storageProvider.name()); fileMetadata.setFileUrl(fileUrl); fileMetadataRepository.save(fileMetadata); diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java index 2a2b310..76cb33f 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java @@ -23,14 +23,14 @@ public class StorageProviderPool { private static final Logger log = LoggerFactory.getLogger(StorageProviderPool.class); - private Map storageProviders = new HashMap<>(); + private Map storageProviders = new HashMap<>(); public StorageProviderPool(GoogleStorageProvider googleStorageProvider, LocalStorageProvider localStorageProvider) { - storageProviders.put("LOCAL", localStorageProvider); - storageProviders.put("GOOGLE", googleStorageProvider); + storageProviders.put(Providers.LOCAL, localStorageProvider); + storageProviders.put(Providers.GOOGLE, googleStorageProvider); } - public StorageProvider getStorageProviderByType(String storageProviderType) { + public StorageProvider getStorageProviderByType(Providers storageProviderType) { log.debug("Get storage provider = {}", storageProviderType); StorageProvider storageProvider = storageProviders.get(storageProviderType); if (storageProvider == null) { diff --git a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java index 4d81a0a..27f9686 100644 --- a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java +++ b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java @@ -7,6 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; +import biz.nynja.content.file.storage.Providers; + /** * @author Ralitsa Todorova * @@ -16,13 +18,18 @@ public class ContentServiceConfiguration { private final String uploadUrl; private final String downloadUrl; - private final String storageProvider; + private final Providers storageProvider; @Autowired public ContentServiceConfiguration(Environment env) { this.uploadUrl = env.getRequiredProperty("file.upload.url"); this.downloadUrl = env.getRequiredProperty("file.download.url"); - this.storageProvider = env.getRequiredProperty("storage.provider"); + this.storageProvider = getProvidersProperty("storage.provider", env); + } + + private Providers getProvidersProperty(String key, Environment env) { + String provider = env.getRequiredProperty("storage.provider"); + return Providers.valueOf(provider); } public String getUploadUrl() { @@ -33,7 +40,7 @@ public class ContentServiceConfiguration { return downloadUrl; } - public String getStorageProvider() { + public Providers getStorageProvider() { return storageProvider; } } diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 722efed..9b4402e 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -19,6 +19,7 @@ import org.springframework.util.StringUtils; import biz.nynja.content.core.validation.Validation; import biz.nynja.content.file.metadata.FileMetadataService; import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.Providers; import biz.nynja.content.file.storage.StorageProvider; import biz.nynja.content.file.storage.StorageProviderPool; import biz.nynja.content.grpc.ContentServiceGrpc; @@ -148,6 +149,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas private int receivedBytesCount; private String fileLocation; private int reachedSize; + private Providers provider = contentSreviceConfiguration.getStorageProvider(); public UploadStreamObserver(StreamObserver responseObserver) { this.responseObserver = responseObserver; @@ -168,7 +170,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas try { if (storageProvider == null) { storageProvider = storageProviderPool - .getStorageProviderByType(contentSreviceConfiguration.getStorageProvider()); + .getStorageProviderByType(provider); fileName = constructFileName(uploadInfo.getFileName()); fileLocation = storageProvider.initialize(fileName); uploadInfo.setStatus("IN_PROGRESS"); @@ -213,7 +215,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas Status.OUT_OF_RANGE.withDescription("Uploaded file size does not meet expectations."))); return; } - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, fileLocation); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, fileLocation, provider); responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); -- GitLab From 4f20cf50c6f5a23c9969ee22b43e2d2c11753d7f Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 11 Jan 2019 17:26:03 +0200 Subject: [PATCH 02/29] NY-6318 Update file metadata - add storage providers; - add file download url generate; - update configuration files; Signed-off-by: abotev-intracol --- .../biz/nynja/content/file/storage/Providers.java | 12 ++++++++++++ .../nynja/content/file/storage/StorageProvider.java | 1 + .../file/storage/impl/GoogleStorageProvider.java | 8 ++++++++ .../file/storage/impl/LocalStorageProvider.java | 5 +++++ .../content/grpc/services/ContentServiceImpl.java | 2 +- src/main/resources/application-dev.yml | 2 +- src/main/resources/application-production.yml | 2 +- 7 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 src/main/java/biz/nynja/content/file/storage/Providers.java diff --git a/src/main/java/biz/nynja/content/file/storage/Providers.java b/src/main/java/biz/nynja/content/file/storage/Providers.java new file mode 100644 index 0000000..55964cc --- /dev/null +++ b/src/main/java/biz/nynja/content/file/storage/Providers.java @@ -0,0 +1,12 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.storage; + +/** + * @author Angel.Botev + * + */ +public enum Providers { + GOOGLE, LOCAL +} diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index 22b6990..cc4df34 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -15,4 +15,5 @@ public interface StorageProvider { public void close(String fileLocation); + public String getFileUrl(String filename); } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index f677152..785df48 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -160,4 +160,12 @@ public class GoogleStorageProvider implements StorageProvider { HttpResponse resp = req.execute(); return resp; } + + @Override + public String getFileUrl(String filename) { + StringBuilder builder = new StringBuilder(); + builder.append(storageConfiguration.getGoogleStorageURI()).append("/") + .append(storageConfiguration.getGoogleBucketName()).append("/").append(filename); + return builder.toString(); + } } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index b9df377..71573d4 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -58,4 +58,9 @@ public class LocalStorageProvider implements StorageProvider { return new StringBuilder(contentStoreLocation).append(File.separator).append(fileName).toString(); } + @Override + public String getFileUrl(String filename) { + return constructFilePath(storageConfiguration.getLocalStorageLocation(), filename); + } + } diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 9b4402e..10295d1 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -215,7 +215,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas Status.OUT_OF_RANGE.withDescription("Uploaded file size does not meet expectations."))); return; } - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, fileLocation, provider); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, storageProvider.getFileUrl(fileName), provider); responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index bf84675..a54500f 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -42,7 +42,7 @@ file: storage: provider: GOOGLE local: - location: src/main/resources/ + location: src/main/resources google: uri: https://storage.googleapis.com bucket: content-service-dev diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 5418080..a87b0e9 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -42,7 +42,7 @@ file: storage: provider: GOOGLE local: - location: ${LOCAL_STORAGE_LOCATION:/src/main/resources/} + location: ${LOCAL_STORAGE_LOCATION:/src/main/resources} google: uri: ${GOOGLE_STORAGE_URI:https://storage.googleapis.com} bucket: ${GOOGLE_STORAGE_BUCKET:content-service-dev} -- GitLab From 11ba2db54c2dc1e8df423c383de3b9c8c1438c8f Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Mon, 14 Jan 2019 14:51:44 +0200 Subject: [PATCH 03/29] NY-6318 Update file metadata - fix comments; Signed-off-by: abotev-intracol --- .../content/file/metadata/FileMetadataService.java | 4 ++-- .../file/storage/{Providers.java => Provider.java} | 2 +- .../content/file/storage/StorageProviderPool.java | 8 ++++---- .../configuration/ContentServiceConfiguration.java | 12 ++++++------ .../content/grpc/services/ContentServiceImpl.java | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) rename src/main/java/biz/nynja/content/file/storage/{Providers.java => Provider.java} (86%) diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index c84e17f..981dff1 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.metadata.repositories.FileMetadataReposiory; -import biz.nynja.content.file.storage.Providers; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.upload.models.UploadStore; /** @@ -30,7 +30,7 @@ public class FileMetadataService { this.fileMetadataRepository = fileMetadataRepository; } - public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl, Providers storageProvider) { + public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl, Provider storageProvider) { FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setKey(UUID.randomUUID()); fileMetadata.setFileName(fileName); diff --git a/src/main/java/biz/nynja/content/file/storage/Providers.java b/src/main/java/biz/nynja/content/file/storage/Provider.java similarity index 86% rename from src/main/java/biz/nynja/content/file/storage/Providers.java rename to src/main/java/biz/nynja/content/file/storage/Provider.java index 55964cc..3435845 100644 --- a/src/main/java/biz/nynja/content/file/storage/Providers.java +++ b/src/main/java/biz/nynja/content/file/storage/Provider.java @@ -7,6 +7,6 @@ package biz.nynja.content.file.storage; * @author Angel.Botev * */ -public enum Providers { +public enum Provider { GOOGLE, LOCAL } diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java index 76cb33f..461e034 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java @@ -23,14 +23,14 @@ public class StorageProviderPool { private static final Logger log = LoggerFactory.getLogger(StorageProviderPool.class); - private Map storageProviders = new HashMap<>(); + private Map storageProviders = new HashMap<>(); public StorageProviderPool(GoogleStorageProvider googleStorageProvider, LocalStorageProvider localStorageProvider) { - storageProviders.put(Providers.LOCAL, localStorageProvider); - storageProviders.put(Providers.GOOGLE, googleStorageProvider); + storageProviders.put(Provider.LOCAL, localStorageProvider); + storageProviders.put(Provider.GOOGLE, googleStorageProvider); } - public StorageProvider getStorageProviderByType(Providers storageProviderType) { + public StorageProvider getStorageProviderByType(Provider storageProviderType) { log.debug("Get storage provider = {}", storageProviderType); StorageProvider storageProvider = storageProviders.get(storageProviderType); if (storageProvider == null) { diff --git a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java index 27f9686..e520288 100644 --- a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java +++ b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java @@ -7,7 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; -import biz.nynja.content.file.storage.Providers; +import biz.nynja.content.file.storage.Provider; /** * @author Ralitsa Todorova @@ -18,7 +18,7 @@ public class ContentServiceConfiguration { private final String uploadUrl; private final String downloadUrl; - private final Providers storageProvider; + private final Provider storageProvider; @Autowired public ContentServiceConfiguration(Environment env) { @@ -27,9 +27,9 @@ public class ContentServiceConfiguration { this.storageProvider = getProvidersProperty("storage.provider", env); } - private Providers getProvidersProperty(String key, Environment env) { - String provider = env.getRequiredProperty("storage.provider"); - return Providers.valueOf(provider); + private Provider getProvidersProperty(String key, Environment env) { + String provider = env.getRequiredProperty(key); + return Provider.valueOf(provider); } public String getUploadUrl() { @@ -40,7 +40,7 @@ public class ContentServiceConfiguration { return downloadUrl; } - public Providers getStorageProvider() { + public Provider getStorageProvider() { return storageProvider; } } diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 10295d1..1e4f7db 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -19,7 +19,7 @@ import org.springframework.util.StringUtils; import biz.nynja.content.core.validation.Validation; import biz.nynja.content.file.metadata.FileMetadataService; import biz.nynja.content.file.metadata.dto.FileMetadata; -import biz.nynja.content.file.storage.Providers; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageProvider; import biz.nynja.content.file.storage.StorageProviderPool; import biz.nynja.content.grpc.ContentServiceGrpc; @@ -149,7 +149,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas private int receivedBytesCount; private String fileLocation; private int reachedSize; - private Providers provider = contentSreviceConfiguration.getStorageProvider(); + private Provider provider = contentSreviceConfiguration.getStorageProvider(); public UploadStreamObserver(StreamObserver responseObserver) { this.responseObserver = responseObserver; -- GitLab From 72c4b9cf4b6336ae5c8124aef55e15e4b0adf60b Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 15 Jan 2019 14:57:40 +0200 Subject: [PATCH 04/29] NY-6494 Extract keyspace replica factor - add replication configuration and add property in CassandraConfig; Signed-off-by: abotev-intracol --- .../nynja/content/db/configuration/CassandraConfig.java | 9 ++++++++- src/main/resources/application-dev.yml | 1 + src/main/resources/application-production.yml | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java index 525af77..fd051ff 100644 --- a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java +++ b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java @@ -22,6 +22,9 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Value("${spring.data.cassandra.keyspace-name}") private String keyspace; + @Value("${spring.data.cassandra.replication}") + private int replication; + @Override protected String getKeyspaceName() { return keyspace; @@ -43,6 +46,10 @@ public class CassandraConfig extends AbstractCassandraConfiguration { return port; } + private int getReplication() { + return replication; + } + @Override public SchemaAction getSchemaAction() { return SchemaAction.CREATE_IF_NOT_EXISTS; @@ -51,7 +58,7 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Override protected List getKeyspaceCreations() { CreateKeyspaceSpecification specification = CreateKeyspaceSpecification.createKeyspace(getKeyspaceName()) - .ifNotExists().withSimpleReplication(); + .ifNotExists().withSimpleReplication(getReplication()); return Arrays.asList(specification); } diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index a54500f..6da8f26 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -15,6 +15,7 @@ spring: keyspace-name: content contact-points: localhost port: 9042 + replication: 3 token: encryptdecrypt: diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index a87b0e9..b2e6ee8 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -15,6 +15,7 @@ spring: keyspace-name: ${CASSANDRA_KEYSPACE:content} contact-points: ${CASSANDRA_CONTACT_POINTS:cassandra.cassandra.svc.cluster.local} port: ${CASSANDRA_PORT:9042} + replication: ${CASSANDRA_KEYSPACE_REPLICATION:3} token: encryptdecrypt: -- GitLab From 6f60487da7c0f00bd8d54107e93c2ed115fa5e54 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 16 Jan 2019 18:14:09 +0200 Subject: [PATCH 05/29] NY-6725 Remove unnecessary logger - replace Groovy library with Spring default; Signed-off-by: abotev-intracol --- pom.xml | 24 ++++++++++---- src/main/resources/application-dev.yml | 32 +++++++++++++++++++ src/main/resources/application-production.yml | 32 +++++++++++++++++++ src/main/resources/logback-spring.groovy | 32 ------------------- 4 files changed, 81 insertions(+), 39 deletions(-) delete mode 100644 src/main/resources/logback-spring.groovy diff --git a/pom.xml b/pom.xml index 15468fb..bc4e4ea 100644 --- a/pom.xml +++ b/pom.xml @@ -124,8 +124,18 @@ - org.codehaus.groovy - groovy-all + org.springframework.boot + spring-boot-starter-actuator + + + + io.micrometer + micrometer-core + + + + io.micrometer + micrometer-registry-prometheus @@ -152,11 +162,11 @@ true - - com.google.cloud - google-cloud-storage - 1.53.0 - + + com.google.cloud + google-cloud-storage + 1.53.0 + diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 6da8f26..4fc172e 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -48,3 +48,35 @@ storage: uri: https://storage.googleapis.com bucket: content-service-dev upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + +# To enable colors in Eclipse: +# spring.output.ansi.enabled=ALWAYS and in eclipse +# Help -> Install New Software... and click "Add..." to add the following URL: +# http://www.mihai-nita.net/eclipse + output: + ansi: + enabled: ALWAYS + +logging: + level: + root: INFO + org: + springframework: + web: INFO + +#Metrics related configurations +management: + endpoint: + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + exposure: + include: 'prometheus, health, info, loggers' + metrics: + export: + prometheus: + enabled: true + diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index b2e6ee8..63f2d69 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -48,3 +48,35 @@ storage: uri: ${GOOGLE_STORAGE_URI:https://storage.googleapis.com} bucket: ${GOOGLE_STORAGE_BUCKET:content-service-dev} upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + +# To enable colors in Eclipse: +# spring.output.ansi.enabled=ALWAYS and in eclipse +# Help -> Install New Software... and click "Add..." to add the following URL: +# http://www.mihai-nita.net/eclipse + output: + ansi: + enabled: ALWAYS + +logging: + level: + root: INFO + org: + springframework: + web: INFO + +#Metrics related configurations +management: + endpoint: + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + exposure: + include: 'prometheus, health, info, loggers' + metrics: + export: + prometheus: + enabled: true + diff --git a/src/main/resources/logback-spring.groovy b/src/main/resources/logback-spring.groovy deleted file mode 100644 index 91261d1..0000000 --- a/src/main/resources/logback-spring.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ - -import ch.qos.logback.classic.encoder.PatternLayoutEncoder -import ch.qos.logback.core.rolling.RollingFileAppender -import ch.qos.logback.core.rolling.TimeBasedRollingPolicy -import ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP -import ch.qos.logback.core.status.OnConsoleStatusListener -import ch.qos.logback.classic.Level - -statusListener(OnConsoleStatusListener) - -def file = "${System.getProperty('log.dir', '')}content-service-%d.%i.log" - -appender("FILE", RollingFileAppender) { - // add a status message regarding the file property - addInfo("Starting logging to $file") - append = true - encoder(PatternLayoutEncoder) { pattern = "%d{HH:mm:ss.SSS} %level %logger - %msg%n" } - rollingPolicy(TimeBasedRollingPolicy) { - fileNamePattern = "$file" - timeBasedFileNamingAndTriggeringPolicy(SizeAndTimeBasedFNATP) { maxFileSize = "10mb" }} -} - -appender("Console-Appender", ConsoleAppender) { - encoder(PatternLayoutEncoder) { pattern = "%d{HH:mm:ss.SSS} %level %logger - %msg%n" } -} - -logger("org.springframework.cloud.config.client.ConfigServicePropertySourceLocator", Level.WARN) -root(INFO, ["Console-Appender"]) -root(Level.toLevel("${System.getProperty('log.level', 'INFO')}"), ["FILE"]) \ No newline at end of file -- GitLab From 8f10ee8614a24bb87769e8c147ecc4199eb2948b Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Thu, 17 Jan 2019 16:21:31 +0200 Subject: [PATCH 06/29] Add maven skipTests=true property. --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cc232c3..9663d5d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } } } @@ -83,7 +83,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } dockerBuildAndPushToRegistry "${NAMESPACE}/${APP_NAME}", [IMAGE_BUILD_TAG] } -- GitLab From 63b5b5d98bced50c7b7880fe5b5dac77c75032b9 Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Fri, 18 Jan 2019 14:06:25 +0200 Subject: [PATCH 07/29] Add skipTests-true property for master and pr-* branches --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9663d5d..cfa5122 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,7 +44,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } } } @@ -120,7 +120,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } dockerBuildAndPushToRegistry "${NAMESPACE}/${APP_NAME}", [IMAGE_BUILD_TAG] } -- GitLab From 1f8282a814c204da6fab485769bf3156d1c5fc18 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Thu, 17 Jan 2019 19:23:56 +0200 Subject: [PATCH 08/29] Change request mapping name in download controller Signed-off-by: Ralitsa Todorova --- .../nynja/content/file/download/FileDownloadController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java index 1af8123..5933218 100644 --- a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java +++ b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java @@ -32,7 +32,7 @@ import com.nimbusds.jose.util.Base64URL; */ @RestController -@RequestMapping("file/") +@RequestMapping("file/download") public class FileDownloadController { private static final Logger logger = LoggerFactory.getLogger(FileDownloadController.class); @@ -44,7 +44,7 @@ public class FileDownloadController { this.fileDownloadService = fileDownloadService; } - @RequestMapping(method = RequestMethod.GET, value = "download/{fileKey}") + @RequestMapping(method = RequestMethod.GET, value = "/{fileKey}") public ResponseEntity downloadFile(@PathVariable String fileKey, HttpServletRequest request) throws FileNotFoundException { logger.info("Downloading file: {} ...", fileKey); -- GitLab From 28b5aef70e7ff3d812738c8926888155efd45d73 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Thu, 17 Jan 2019 19:27:04 +0200 Subject: [PATCH 09/29] NY-6474: Introduce UploadStatus enum Signed-off-by: Ralitsa Todorova --- .../content/grpc/services/ContentServiceImpl.java | 10 ++++++---- .../nynja/content/upload/models/UploadStatus.java | 13 +++++++++++++ .../nynja/content/upload/models/UploadStore.java | 11 ++++------- 3 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 src/main/java/biz/nynja/content/upload/models/UploadStatus.java diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 1e4f7db..629a518 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -31,6 +31,7 @@ import biz.nynja.content.grpc.UploadTokenResponse; import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; import biz.nynja.content.grpc.constants.ContentServiceConstants; import biz.nynja.content.grpc.interceptors.UploadInterceptor; +import biz.nynja.content.upload.models.UploadStatus; import biz.nynja.content.upload.models.UploadStore; import biz.nynja.content.upload.token.UploadToken; import biz.nynja.content.upload.token.UploadTokenResponseProvider; @@ -109,11 +110,12 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas logger.error("Error generating upload token:{} ", token.toString()); return; } + UUID jobId = UUID.randomUUID(); UploadStore uploadInfo = new UploadStore(tokenResponse.get(), Instant.now().toEpochMilli() + uploadTokenConfiguration.getTokenTimeToLive(), - contentSreviceConfiguration.getUploadUrl(), UUID.randomUUID(), request.getDeviceId(), + contentSreviceConfiguration.getUploadUrl() + jobId, jobId, request.getDeviceId(), UUID.fromString(accountId), request.getName(), request.getSize(), request.getMediaType().name(), - request.getUploadedFrom(), "PENDING"); + request.getUploadedFrom(), UploadStatus.PENDING); if (!uploadTokenService.storeToken(uploadInfo, false)) { uploadTokenResponseProvider.prepareErrorResponse(responseObserver, Cause.INTERNAL_SERVER_ERROR, @@ -173,7 +175,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas .getStorageProviderByType(provider); fileName = constructFileName(uploadInfo.getFileName()); fileLocation = storageProvider.initialize(fileName); - uploadInfo.setStatus("IN_PROGRESS"); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); if (!uploadTokenService.storeToken(uploadInfo, true)) { handleStreamTermination(); responseObserver.onError(new StatusException( @@ -226,7 +228,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas private void handleStreamTermination() { uploadInfo.setRetries(uploadInfo.getRetries() + 1); - uploadInfo.setStatus("FAILED"); + uploadInfo.setStatus(UploadStatus.FAILED); uploadTokenService.storeToken(uploadInfo, false); storageProvider.close(fileLocation); } diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java b/src/main/java/biz/nynja/content/upload/models/UploadStatus.java new file mode 100644 index 0000000..d0380b5 --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/models/UploadStatus.java @@ -0,0 +1,13 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.models; + +/** + * @author Ralitsa Todorova + * + */ +public enum UploadStatus { + + PENDING, IN_PROGRESS, FAILED; +} diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/upload/models/UploadStore.java index 0cc0756..4df158f 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/upload/models/UploadStore.java @@ -26,7 +26,7 @@ public class UploadStore { private int fileSize; private String mediaType; private String uploadedFrom; - private String status; + private UploadStatus status; private int retries; public UploadStore() { @@ -116,11 +116,11 @@ public class UploadStore { this.uploadedFrom = uploadedFrom; } - public String getStatus() { + public UploadStatus getStatus() { return status; } - public void setStatus(String status) { + public void setStatus(UploadStatus status) { this.status = status; } @@ -201,10 +201,7 @@ public class UploadStore { return false; if (retries != other.retries) return false; - if (status == null) { - if (other.status != null) - return false; - } else if (!status.equals(other.status)) + if (status != other.status) return false; if (tokenExpirationTime == null) { if (other.tokenExpirationTime != null) -- GitLab From 610ea5b31e0ff6d380b495b453486221092821b8 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Thu, 17 Jan 2019 19:29:28 +0200 Subject: [PATCH 10/29] NY-6474: Extend UploadStore model - added partId and lastUploadedByte fields Signed-off-by: Ralitsa Todorova --- .../content/upload/models/UploadStore.java | 31 +++++++++++++++++-- .../CustomizedUploadStoreRepositoryImpl.java | 9 +++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/upload/models/UploadStore.java index 4df158f..ec9c10a 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/upload/models/UploadStore.java @@ -28,12 +28,14 @@ public class UploadStore { private String uploadedFrom; private UploadStatus status; private int retries; + private int partId; + private int lastUploadedByte; public UploadStore() { } public UploadStore(String uploadToken, Long tokenExpirationTime, String uploadUrl, UUID jobId, String deviceId, - UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, String status) { + UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, UploadStatus status) { this.uploadToken = uploadToken; this.tokenExpirationTime = tokenExpirationTime; this.uploadUrl = uploadUrl; @@ -46,6 +48,8 @@ public class UploadStore { this.uploadedFrom = uploadedFrom; this.status = status; this.retries = 0; + this.partId = 0; + this.lastUploadedByte = 0; } public String getUploadToken() { @@ -144,6 +148,22 @@ public class UploadStore { this.retries = retries; } + public int getPartId() { + return partId; + } + + public void setPartId(int partId) { + this.partId = partId; + } + + public int getLastUploadedByte() { + return lastUploadedByte; + } + + public void setLastUploadedByte(int lastUploadedByte) { + this.lastUploadedByte = lastUploadedByte; + } + @Override public int hashCode() { final int prime = 31; @@ -153,7 +173,9 @@ public class UploadStore { result = prime * result + ((fileName == null) ? 0 : fileName.hashCode()); result = prime * result + fileSize; result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); + result = prime * result + lastUploadedByte; result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); + result = prime * result + partId; result = prime * result + retries; result = prime * result + ((status == null) ? 0 : status.hashCode()); result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); @@ -194,11 +216,15 @@ public class UploadStore { return false; } else if (!jobId.equals(other.jobId)) return false; + if (lastUploadedByte != other.lastUploadedByte) + return false; if (mediaType == null) { if (other.mediaType != null) return false; } else if (!mediaType.equals(other.mediaType)) return false; + if (partId != other.partId) + return false; if (retries != other.retries) return false; if (status != other.status) @@ -233,7 +259,8 @@ public class UploadStore { .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") - .append(retries).append("]").toString(); + .append(retries).append(", partId=").append(partId).append(", lastUploadedByte=") + .append(lastUploadedByte).append("]").toString(); } } diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java b/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java index c2caaaa..e82580e 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java +++ b/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java @@ -35,13 +35,14 @@ public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadS } private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { - String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + ttl + ";"; return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), - uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), uploadStore.getStatus(), - uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), uploadStore.getUploadUrl(), - uploadStore.getRetries()); + uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), + uploadStore.getStatus().toString(), uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), + uploadStore.getUploadUrl(), uploadStore.getRetries(), uploadStore.getPartId(), + uploadStore.getLastUploadedByte()); } } -- GitLab From 27db02f42ffda22a382702d016b88af380ea55d9 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Thu, 17 Jan 2019 19:32:47 +0200 Subject: [PATCH 11/29] NY-6474: Implement REST endpoint for resumable file upload on chunks Signed-off-by: Ralitsa Todorova --- .../storage/impl/LocalStorageProvider.java | 8 +- .../upload/rest/FileUploadController.java | 157 ++++++++++++++++++ .../rest/validation/ChunkUploadValidator.java | 74 +++++++++ .../rest/validation/ValidationResult.java | 26 +++ src/main/resources/application-dev.yml | 2 +- src/main/resources/application-production.yml | 2 +- 6 files changed, 263 insertions(+), 6 deletions(-) create mode 100644 src/main/java/biz/nynja/content/upload/rest/FileUploadController.java create mode 100644 src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java create mode 100644 src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 71573d4..0aa9656 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -40,13 +40,13 @@ public class LocalStorageProvider implements StorageProvider { @Override public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws IOException { - logger.debug("Appending bytes in file: {}", fileLocation); + logger.debug("Appending bytes to file: {}", fileLocation); FileOutputStream output = new FileOutputStream(fileLocation, true); try { - output.write(data); - logger.info("Bytes was successfully append to file {}.", fileLocation); + output.write(data); + logger.info("Bytes were successfully append to file {}.", fileLocation); } finally { - output.close(); + output.close(); } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java new file mode 100644 index 0000000..40894bf --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -0,0 +1,157 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest; + +import java.util.Optional; + +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import biz.nynja.content.file.metadata.FileMetadataService; +import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.storage.StorageProviderPool; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; +import biz.nynja.content.upload.models.UploadStatus; +import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.upload.rest.validation.ValidationResult; +import biz.nynja.content.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@RestController +@RequestMapping("/file/upload") +public class FileUploadController { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); + + private StorageProvider storageProvider; + private StorageProviderPool storageProviderPool; + private ContentServiceConfiguration contentServiceConfiguration; + private UploadTokenService uploadTokenService; + private FileMetadataService fileMetadataService; + private ChunkUploadValidator validator; + + public FileUploadController(StorageProviderPool storageProviderPool, + ContentServiceConfiguration contentServiceConfiguration, UploadTokenService uploadTokenService, + FileMetadataService fileMetadataService, ChunkUploadValidator validator) { + this.storageProviderPool = storageProviderPool; + this.contentServiceConfiguration = contentServiceConfiguration; + this.uploadTokenService = uploadTokenService; + this.fileMetadataService = fileMetadataService; + this.validator = validator; + } + + @PutMapping("/{jobId}") + public ResponseEntity uploadFile(@PathVariable String jobId, @RequestParam("token") String uploadToken, + @RequestParam("partId") int partId, @RequestParam("last") boolean last, HttpServletRequest request, + @RequestBody byte[] dataChunk) { + + logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); + UploadStore uploadInfo; + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (uploadInfoResult.isPresent()) { + uploadInfo = uploadInfoResult.get(); + } else { + logger.error("You are not allowed to upload data. Provided token is either wrong or missing."); + return new ResponseEntity<>( + "You are not allowed to upload data. Provided token is either wrong or missing.", + HttpStatus.BAD_REQUEST); + } + + Optional requestValidation = validator.validateUploadRequest(uploadInfo, dataChunk, jobId, + uploadToken,partId, last); + if (requestValidation.isPresent()) { + return new ResponseEntity<>(requestValidation.get().getMessage(), HttpStatus.BAD_REQUEST); + } + + String fileName = retrieveFileName(uploadInfo); + String fileLocation; + + try { + if (storageProvider == null) { + storageProvider = storageProviderPool + .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); + } + + fileLocation = storageProvider.initialize(fileName); + logger.info("Created output stream for file {}", fileLocation); + + storageProvider.write(dataChunk, fileLocation, partId, last); + logger.info("Chunk {} was successfully written to output stream {}.", partId, fileName); + uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); + if (!uploadTokenService.storeToken(uploadInfo, true)) { + // In this case data chunk was successfully appended to file (when LOCAL storage is used), but + // corresponding DB record contains incorrect data. This situation leads to data inconsistency. + // TODO: Check this behavior against cloud storage. (Google Cloud Storage). + logger.error( + "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); + return new ResponseEntity<>( + "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", + HttpStatus.INTERNAL_SERVER_ERROR); + } + } catch (Exception e) { + logger.error("Error writing data for file {}: {}", fileName, e.getMessage()); + logger.debug("Error writing data for file {}: {}", fileName, e.getCause()); + return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); + } + if (last) { + finalizeUpload(uploadInfo, fileLocation); + } + return new ResponseEntity<>(HttpStatus.CREATED); + } + + private String constructFileName(String fileName) { + String generatedPrefix = RandomStringUtils.randomAlphabetic(10); + return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); + } + + private String constructDownloadUrl(String fileKey) { + return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); + } + + private String retrieveFileName(UploadStore uploadInfo) { + String fileName; + if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // upload hasn't started yet + fileName = constructFileName(uploadInfo.getFileName()); + uploadInfo.setFileName(fileName); + } else { + fileName = uploadInfo.getFileName(); + } + return fileName; + } + + private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { + uploadInfo.setPartId(partId); + uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); + return uploadInfo; + + } + + private String finalizeUpload(UploadStore uploadInfo, String fileLocation) { + + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, uploadInfo.getFileName(), + fileLocation); + String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); + storageProvider.close(fileLocation); + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + logger.info("File sucessfully uploaded."); + return dowlnoadUrl; + } +} diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java b/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java new file mode 100644 index 0000000..6e325e2 --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest.validation; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import biz.nynja.content.upload.models.UploadStore; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class ChunkUploadValidator { + + private static final Logger logger = LoggerFactory.getLogger(ChunkUploadValidator.class); + + public Optional validateUploadRequest(UploadStore uploadInfo, byte[] data, String jobId, + String token, int partId, boolean last) { + + if (!jobId.equals(uploadInfo.getJobId().toString())) { + logger.error("Upload token {} does not apply to the requested job id: {}", token, jobId); + return Optional.of(new ValidationResult("Upload token does not apply to the requested upload URL.")); + } + + if (partId != uploadInfo.getPartId() + 1) { + logger.error( + "Chunk with Unexpected part id received for job id {} with associated token: {} . Expected: {}, actual: {}", + jobId, token, uploadInfo.getPartId() + 1, partId); + return Optional.of(new ValidationResult( + "You are trying to upload wrong data chunk. Last uploaded chunk " + uploadInfo.getPartId())); + } + if (data.length <= 0) { + logger.error("Recieved empty data chuck for jobId {} with associated token: {}", jobId, token); + return Optional.of(new ValidationResult("Missing chunk data.")); + } + + String sizeValidationResult = validateReachedFileSize(uploadInfo.getLastUploadedByte() + data.length, + uploadInfo.getFileSize(), last); + if (sizeValidationResult != null) { + logger.error("Incorrect file size reached for jobId {} with associated token: {}. {}", jobId, token, + sizeValidationResult); + return Optional.of(new ValidationResult(sizeValidationResult)); + } + + return Optional.empty(); + } + + private boolean matchesExpectedSize(int actual, int expected) { + return actual == expected; + } + + private boolean exceedsExpectedSize(int actual, int expected) { + return actual > expected; + } + + private String validateReachedFileSize(int actual, int expected, boolean lastChunk) { + if (exceedsExpectedSize(actual, expected)) { + logger.error("Size of uploaded file so far exceeds the expected file size."); + return "Size of uploaded file so far exceeds the expected file size."; + } + if (lastChunk && !matchesExpectedSize(actual, expected)) { + logger.error("Uploaded file size does not meet expectations - expected: {}, actual: {}.", expected, actual); + return "Uploaded file size does not meet expectations."; + } + return null; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java b/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java new file mode 100644 index 0000000..5074bd9 --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest.validation; + +/** + * @author Ralitsa Todorova + * + */ +public class ValidationResult { + + private String message; + + public ValidationResult(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + +} diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 4fc172e..a4cfd5b 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -36,7 +36,7 @@ media-types: file: upload: - url: http://localhost:${grpc.port} + url: http://localhost:${grpc.port}/file/upload/ download: url: http://localhost:${server.port}/file/download/ diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 63f2d69..29d8d6c 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -36,7 +36,7 @@ media-types: file: upload: - url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net}:${grpc.port} + url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net/file/upload} download: url: ${FILE_DOWNLOAD_URL:https://content.dev-eu.nynja.net/file/download/} -- GitLab From 89459dab779f59948fe15684b6383bb56c6d877d Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 11:32:02 +0200 Subject: [PATCH 12/29] NY-6474: Fixed wording Signed-off-by: Ralitsa Todorova --- .../nynja/content/file/storage/impl/LocalStorageProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 0aa9656..e333ee6 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -44,7 +44,7 @@ public class LocalStorageProvider implements StorageProvider { FileOutputStream output = new FileOutputStream(fileLocation, true); try { output.write(data); - logger.info("Bytes were successfully append to file {}.", fileLocation); + logger.info("Bytes were successfully appended to file {}.", fileLocation); } finally { output.close(); } -- GitLab From 44a92487cafd667a1e22f3747549eeb2103e00b7 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 17:52:27 +0200 Subject: [PATCH 13/29] NY-6474: Add getProviderType method for Storage Provider classes Signed-off-by: Ralitsa Todorova --- .../biz/nynja/content/file/storage/StorageProvider.java | 2 ++ .../content/file/storage/impl/GoogleStorageProvider.java | 8 ++++++++ .../content/file/storage/impl/LocalStorageProvider.java | 8 ++++++++ 3 files changed, 18 insertions(+) diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index cc4df34..c62c516 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -16,4 +16,6 @@ public interface StorageProvider { public void close(String fileLocation); public String getFileUrl(String filename); + + public Provider getProviderType(); } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 785df48..0d971e7 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -28,6 +28,7 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.InputStreamContent; import com.google.api.services.storage.StorageScopes; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageConfiguration; import biz.nynja.content.file.storage.StorageProvider; import io.netty.buffer.ByteBufInputStream; @@ -43,6 +44,8 @@ public class GoogleStorageProvider implements StorageProvider { private static final Logger logger = LoggerFactory.getLogger(GoogleStorageProvider.class); + private static final Provider providerType = Provider.GOOGLE; + private HttpRequestFactory requestFactory; private StorageConfiguration storageConfiguration; @@ -168,4 +171,9 @@ public class GoogleStorageProvider implements StorageProvider { .append(storageConfiguration.getGoogleBucketName()).append("/").append(filename); return builder.toString(); } + + @Override + public Provider getProviderType() { + return providerType; + } } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index e333ee6..8d4bcbb 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -12,6 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageConfiguration; import biz.nynja.content.file.storage.StorageProvider; @@ -25,6 +26,8 @@ public class LocalStorageProvider implements StorageProvider { private static final Logger logger = LoggerFactory.getLogger(LocalStorageProvider.class); + private static final Provider providerType = Provider.LOCAL; + private StorageConfiguration storageConfiguration; public LocalStorageProvider(StorageConfiguration storageConfiguration) { @@ -63,4 +66,9 @@ public class LocalStorageProvider implements StorageProvider { return constructFilePath(storageConfiguration.getLocalStorageLocation(), filename); } + @Override + public Provider getProviderType() { + return providerType; + } + } -- GitLab From 6789d134326ea4e8cc3efd072c34db492d5320b3 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 17:53:17 +0200 Subject: [PATCH 14/29] NY-6474: Set correct port in file uplaod URL Signed-off-by: Ralitsa Todorova --- src/main/resources/application-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index a4cfd5b..77123f4 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -36,7 +36,7 @@ media-types: file: upload: - url: http://localhost:${grpc.port}/file/upload/ + url: http://localhost:${server.port}/file/upload/ download: url: http://localhost:${server.port}/file/download/ -- GitLab From 6ab48d95b7e391b1267c81b7118f2a52f7d19f81 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 17:55:58 +0200 Subject: [PATCH 15/29] NY-6474: Improve file metadata storing Signed-off-by: Ralitsa Todorova --- .../nynja/content/file/metadata/FileMetadataService.java | 4 ++-- .../nynja/content/grpc/services/ContentServiceImpl.java | 3 ++- .../nynja/content/upload/rest/FileUploadController.java | 9 ++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index 981dff1..427f71f 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -30,10 +30,10 @@ public class FileMetadataService { this.fileMetadataRepository = fileMetadataRepository; } - public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl, Provider storageProvider) { + public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileUrl, Provider storageProvider) { FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setKey(UUID.randomUUID()); - fileMetadata.setFileName(fileName); + fileMetadata.setFileName(uploadInfo.getFileName()); fileMetadata.setFileSize(uploadInfo.getFileSize()); fileMetadata.setMediaType(uploadInfo.getMediaType()); fileMetadata.setUploadedFrom(uploadInfo.getUploadedFrom()); diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 629a518..8188c84 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -217,7 +217,8 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas Status.OUT_OF_RANGE.withDescription("Uploaded file size does not meet expectations."))); return; } - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, storageProvider.getFileUrl(fileName), provider); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + storageProvider.getFileUrl(fileName), storageProvider.getProviderType()); responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index 40894bf..bace40e 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -111,7 +111,7 @@ public class FileUploadController { return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); } if (last) { - finalizeUpload(uploadInfo, fileLocation); + finalizeUpload(uploadInfo, storageProvider); } return new ResponseEntity<>(HttpStatus.CREATED); } @@ -144,12 +144,11 @@ public class FileUploadController { } - private String finalizeUpload(UploadStore uploadInfo, String fileLocation) { + private String finalizeUpload(UploadStore uploadInfo, StorageProvider provider) { - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, uploadInfo.getFileName(), - fileLocation); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + provider.getFileUrl(uploadInfo.getFileName()), provider.getProviderType()); String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); - storageProvider.close(fileLocation); uploadTokenService.deleteToken(uploadInfo.getUploadToken()); logger.info("File sucessfully uploaded."); return dowlnoadUrl; -- GitLab From a16e226e2bbf3a6c0995f4c95cd3e7bcd47e71fa Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Fri, 18 Jan 2019 19:20:54 +0200 Subject: [PATCH 16/29] NY-6475: Expect accessToken to be passed as Authorization header Signed-off-by: Ralitsa Todorova --- .../grpc/constants/ContentServiceConstants.java | 2 +- .../interceptors/ContentServiceInterceptor.java | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java b/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java index eef546e..0bb2dd6 100644 --- a/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java +++ b/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java @@ -14,7 +14,7 @@ import io.grpc.Metadata; */ public class ContentServiceConstants { - public static final Metadata.Key ACCESS_TOKEN_METADATA = Metadata.Key.of("accessToken", + public static final Metadata.Key ACCESS_TOKEN_METADATA = Metadata.Key.of("Authorization", ASCII_STRING_MARSHALLER); public static final Context.Key ACCOUNT_ID = Context.key("accountId"); diff --git a/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java b/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java index f67e858..c0f0bfb 100644 --- a/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java +++ b/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java @@ -22,6 +22,7 @@ import io.grpc.ServerCall; import io.grpc.ServerCall.Listener; import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; +import io.grpc.Status; /** * @author Ralitsa Todorova @@ -40,15 +41,26 @@ public class ContentServiceInterceptor implements ServerInterceptor { @Override public Listener interceptCall(ServerCall call, Metadata headers, ServerCallHandler next) { - String accessToken = headers.get(ContentServiceConstants.ACCESS_TOKEN_METADATA); + String accessToken = getAccessToken(headers.get(ContentServiceConstants.ACCESS_TOKEN_METADATA)); Context ctx = Context.current(); if (!StringUtils.isEmpty(accessToken)) { ctx = Context.current().withValue(ContentServiceConstants.ACCOUNT_ID, retrieveAccountId(accessToken)); - } + } else + call.close(Status.PERMISSION_DENIED, new Metadata()); return Contexts.interceptCall(ctx, call, headers, next); } + private String getAccessToken(String authHeader) { + // Authorization header is expected to be in format -> "Authorization" : "Bearer <>" + String accessToken = null; + try { + accessToken = authHeader.split(" ")[1]; + } catch (Exception e) { + logger.error("Unexpected Authorization Header format: {}", authHeader); + } + return accessToken; + } private String retrieveAccountId(String accessToken) { DecodedJWT decodedToken = JWT.decode(accessToken); String accountIdEncoded = decodedToken.getSubject(); -- GitLab From fea6fde6442fba5f455b02c883590b0ec262eb95 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Mon, 21 Jan 2019 18:05:59 +0200 Subject: [PATCH 17/29] NY-6474: Refactor FileUploadController - Move buisness logic in a separate service class - Introduce UploadResponse Signed-off-by: Ralitsa Todorova --- .../content/upload/models/UploadStatus.java | 2 +- .../upload/rest/FileUploadController.java | 112 ++++------------- .../upload/rest/FileUploadService.java | 113 ++++++++++++++++++ .../content/upload/rest/UploadResponse.java | 112 +++++++++++++++++ 4 files changed, 246 insertions(+), 93 deletions(-) create mode 100644 src/main/java/biz/nynja/content/upload/rest/FileUploadService.java create mode 100644 src/main/java/biz/nynja/content/upload/rest/UploadResponse.java diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java b/src/main/java/biz/nynja/content/upload/models/UploadStatus.java index d0380b5..ac9e5e9 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java +++ b/src/main/java/biz/nynja/content/upload/models/UploadStatus.java @@ -9,5 +9,5 @@ package biz.nynja.content.upload.models; */ public enum UploadStatus { - PENDING, IN_PROGRESS, FAILED; + PENDING, IN_PROGRESS, FAILED, COMPLETED; } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index bace40e..107f3e3 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -7,7 +7,6 @@ import java.util.Optional; import javax.servlet.http.HttpServletRequest; -import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -19,12 +18,6 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import biz.nynja.content.file.metadata.FileMetadataService; -import biz.nynja.content.file.metadata.dto.FileMetadata; -import biz.nynja.content.file.storage.StorageProvider; -import biz.nynja.content.file.storage.StorageProviderPool; -import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; -import biz.nynja.content.upload.models.UploadStatus; import biz.nynja.content.upload.models.UploadStore; import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; import biz.nynja.content.upload.rest.validation.ValidationResult; @@ -40,27 +33,22 @@ public class FileUploadController { private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); - private StorageProvider storageProvider; - private StorageProviderPool storageProviderPool; - private ContentServiceConfiguration contentServiceConfiguration; private UploadTokenService uploadTokenService; - private FileMetadataService fileMetadataService; private ChunkUploadValidator validator; + private FileUploadService fileUploadService; + + public FileUploadController(UploadTokenService uploadTokenService, ChunkUploadValidator validator, + FileUploadService fileUploadService) { - public FileUploadController(StorageProviderPool storageProviderPool, - ContentServiceConfiguration contentServiceConfiguration, UploadTokenService uploadTokenService, - FileMetadataService fileMetadataService, ChunkUploadValidator validator) { - this.storageProviderPool = storageProviderPool; - this.contentServiceConfiguration = contentServiceConfiguration; this.uploadTokenService = uploadTokenService; - this.fileMetadataService = fileMetadataService; this.validator = validator; + this.fileUploadService = fileUploadService; } @PutMapping("/{jobId}") - public ResponseEntity uploadFile(@PathVariable String jobId, @RequestParam("token") String uploadToken, - @RequestParam("partId") int partId, @RequestParam("last") boolean last, HttpServletRequest request, - @RequestBody byte[] dataChunk) { + public ResponseEntity uploadFile(@PathVariable String jobId, + @RequestParam("token") String uploadToken, @RequestParam("partId") int partId, + @RequestParam("last") boolean last, HttpServletRequest request, @RequestBody byte[] dataChunk) { logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); UploadStore uploadInfo; @@ -70,87 +58,27 @@ public class FileUploadController { } else { logger.error("You are not allowed to upload data. Provided token is either wrong or missing."); return new ResponseEntity<>( - "You are not allowed to upload data. Provided token is either wrong or missing.", + new UploadResponse( + "You are not allowed to upload data. Provided token is either wrong or missing."), HttpStatus.BAD_REQUEST); } - + Optional requestValidation = validator.validateUploadRequest(uploadInfo, dataChunk, jobId, - uploadToken,partId, last); + uploadToken, partId, last); if (requestValidation.isPresent()) { - return new ResponseEntity<>(requestValidation.get().getMessage(), HttpStatus.BAD_REQUEST); + return new ResponseEntity<>(new UploadResponse(requestValidation.get().getMessage(), uploadInfo.getStatus(), + uploadInfo.getPartId()), HttpStatus.BAD_REQUEST); } - String fileName = retrieveFileName(uploadInfo); - String fileLocation; - try { - if (storageProvider == null) { - storageProvider = storageProviderPool - .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); - } - - fileLocation = storageProvider.initialize(fileName); - logger.info("Created output stream for file {}", fileLocation); - - storageProvider.write(dataChunk, fileLocation, partId, last); - logger.info("Chunk {} was successfully written to output stream {}.", partId, fileName); - uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); - if (!uploadTokenService.storeToken(uploadInfo, true)) { - // In this case data chunk was successfully appended to file (when LOCAL storage is used), but - // corresponding DB record contains incorrect data. This situation leads to data inconsistency. - // TODO: Check this behavior against cloud storage. (Google Cloud Storage). - logger.error( - "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); - return new ResponseEntity<>( - "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", - HttpStatus.INTERNAL_SERVER_ERROR); - } + UploadResponse result = fileUploadService.writeDataChunk(uploadInfo, dataChunk, partId, last); + return new ResponseEntity<>(result, HttpStatus.CREATED); } catch (Exception e) { - logger.error("Error writing data for file {}: {}", fileName, e.getMessage()); - logger.debug("Error writing data for file {}: {}", fileName, e.getCause()); - return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); + logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); + logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); + return new ResponseEntity<>(new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), + uploadInfo.getLastUploadedByte()), HttpStatus.INTERNAL_SERVER_ERROR); } - if (last) { - finalizeUpload(uploadInfo, storageProvider); - } - return new ResponseEntity<>(HttpStatus.CREATED); - } - - private String constructFileName(String fileName) { - String generatedPrefix = RandomStringUtils.randomAlphabetic(10); - return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); - } - - private String constructDownloadUrl(String fileKey) { - return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); } - private String retrieveFileName(UploadStore uploadInfo) { - String fileName; - if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // upload hasn't started yet - fileName = constructFileName(uploadInfo.getFileName()); - uploadInfo.setFileName(fileName); - } else { - fileName = uploadInfo.getFileName(); - } - return fileName; - } - - private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { - uploadInfo.setPartId(partId); - uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); - uploadInfo.setStatus(UploadStatus.IN_PROGRESS); - return uploadInfo; - - } - - private String finalizeUpload(UploadStore uploadInfo, StorageProvider provider) { - - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, - provider.getFileUrl(uploadInfo.getFileName()), provider.getProviderType()); - String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); - uploadTokenService.deleteToken(uploadInfo.getUploadToken()); - logger.info("File sucessfully uploaded."); - return dowlnoadUrl; - } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java new file mode 100644 index 0000000..00fc32e --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest; + +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import biz.nynja.content.file.metadata.FileMetadataService; +import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.storage.StorageProviderPool; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; +import biz.nynja.content.upload.models.UploadStatus; +import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class FileUploadService { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); + + private StorageProvider storageProvider; + private UploadTokenService uploadTokenService; + private FileMetadataService fileMetadataService; + private ContentServiceConfiguration contentServiceConfiguration; + + @Autowired + public FileUploadService(StorageProviderPool storageProviderPool, UploadTokenService uploadTokenService, + FileMetadataService fileMetadataService, ContentServiceConfiguration contentServiceConfiguration) { + this.uploadTokenService = uploadTokenService; + this.fileMetadataService = fileMetadataService; + this.contentServiceConfiguration = contentServiceConfiguration; + this.storageProvider = storageProviderPool + .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); + } + + public UploadResponse writeDataChunk(UploadStore uploadInfo, byte[] dataChunk, int partId, boolean last) + throws Exception { + String fileName = retrieveFileName(uploadInfo); + String fileLocation; + + fileLocation = storageProvider.initialize(fileName); + logger.info("Created output stream for file {}", fileLocation); + + storageProvider.write(dataChunk, fileLocation, partId, last); + logger.info("Chunk {} was successfully written to output stream {}.", partId, fileName); + uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); + if (!uploadTokenService.storeToken(uploadInfo, true)) { + // In this case data chunk was successfully appended to file (when LOCAL storage is used), but + // corresponding DB record contains incorrect data. This situation leads to data inconsistency. + // TODO: Check this behavior against cloud storage. (Google Cloud Storage). + logger.error( + "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); + return new UploadResponse( + "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", + UploadStatus.FAILED, uploadInfo.getPartId()); + + } + if (last) { + finalizeUpload(uploadInfo); + return new UploadResponse("File uplaoad successfully completed.", UploadStatus.COMPLETED, + uploadInfo.getPartId()); + } + + return new UploadResponse("Data chunk uplaoad successfully completed.", UploadStatus.IN_PROGRESS, + uploadInfo.getPartId()); + } + + public String finalizeUpload(UploadStore uploadInfo) { + + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + storageProvider.getFileUrl(uploadInfo.getFileName()), storageProvider.getProviderType()); + String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + logger.info("File sucessfully uploaded."); + return dowlnoadUrl; + } + + private String retrieveFileName(UploadStore uploadInfo) { + String fileName; + if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // upload hasn't started yet + fileName = constructFileName(uploadInfo.getFileName()); + uploadInfo.setFileName(fileName); + } else { + fileName = uploadInfo.getFileName(); + } + return fileName; + } + + private String constructFileName(String fileName) { + String generatedPrefix = RandomStringUtils.randomAlphabetic(10); + return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); + } + + private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { + uploadInfo.setPartId(partId); + uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); + return uploadInfo; + } + + private String constructDownloadUrl(String fileKey) { + return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java b/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java new file mode 100644 index 0000000..0cb0766 --- /dev/null +++ b/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java @@ -0,0 +1,112 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.upload.rest; + +import biz.nynja.content.upload.models.UploadStatus; + +/** + * @author Ralitsa Todorova + * + */ +public class UploadResponse { + + private String message; + private UploadStatus status; + private int lastSuccessfulChunk; + private String downloadLink; + + public UploadResponse(String message) { + this.message = message; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk, String downloadLink) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + this.downloadLink = downloadLink; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public UploadStatus getStatus() { + return status; + } + + public void setStatus(UploadStatus status) { + this.status = status; + } + + public int getLastSuccessfulChunk() { + return lastSuccessfulChunk; + } + + public void setLastSuccessfulChunk(int lastSuccessfulChunk) { + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public String getDownloadLink() { + return downloadLink; + } + + public void setDownloadLink(String downloadLink) { + this.downloadLink = downloadLink; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((downloadLink == null) ? 0 : downloadLink.hashCode()); + result = prime * result + lastSuccessfulChunk; + result = prime * result + ((message == null) ? 0 : message.hashCode()); + result = prime * result + ((status == null) ? 0 : status.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadResponse other = (UploadResponse) obj; + if (downloadLink == null) { + if (other.downloadLink != null) + return false; + } else if (!downloadLink.equals(other.downloadLink)) + return false; + if (lastSuccessfulChunk != other.lastSuccessfulChunk) + return false; + if (message == null) { + if (other.message != null) + return false; + } else if (!message.equals(other.message)) + return false; + if (status != other.status) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadResponse [message=").append(message).append(", status=").append(status) + .append(", lastSuccessfulChunk=").append(lastSuccessfulChunk).append(", downloadLink=") + .append(downloadLink).append("]").toString(); + } + +} -- GitLab From a42211a5ea1479b3e4d22c4f253dbe9fc00da82f Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Tue, 22 Jan 2019 16:35:03 +0200 Subject: [PATCH 18/29] NY-6474: Extend upload info with additional parameters - Add timestamp for last uploaded chunk; - Add uploadLocation Signed-off-by: Ralitsa Todorova --- .../content/upload/models/UploadStore.java | 37 ++++++++++++++++++- .../CustomizedUploadStoreRepositoryImpl.java | 5 ++- .../upload/rest/FileUploadController.java | 2 +- .../upload/rest/FileUploadService.java | 36 +++++++++--------- 4 files changed, 58 insertions(+), 22 deletions(-) diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/upload/models/UploadStore.java index ec9c10a..66f4757 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/upload/models/UploadStore.java @@ -30,6 +30,8 @@ public class UploadStore { private int retries; private int partId; private int lastUploadedByte; + private String uploadLocation; + private Long lastUploadedChunkTimestamp; public UploadStore() { } @@ -50,6 +52,8 @@ public class UploadStore { this.retries = 0; this.partId = 0; this.lastUploadedByte = 0; + this.uploadLocation = ""; + this.lastUploadedChunkTimestamp = 0L; } public String getUploadToken() { @@ -164,6 +168,22 @@ public class UploadStore { this.lastUploadedByte = lastUploadedByte; } + public String getUploadLocation() { + return uploadLocation; + } + + public void setUploadLocation(String uploadLocation) { + this.uploadLocation = uploadLocation; + } + + public Long getLastUploadedChunkTimestamp() { + return lastUploadedChunkTimestamp; + } + + public void setLastUploadedChunkTimestamp(Long lastUploadedChunkTimestamp) { + this.lastUploadedChunkTimestamp = lastUploadedChunkTimestamp; + } + @Override public int hashCode() { final int prime = 31; @@ -174,11 +194,13 @@ public class UploadStore { result = prime * result + fileSize; result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); result = prime * result + lastUploadedByte; + result = prime * result + ((lastUploadedChunkTimestamp == null) ? 0 : lastUploadedChunkTimestamp.hashCode()); result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); result = prime * result + partId; result = prime * result + retries; result = prime * result + ((status == null) ? 0 : status.hashCode()); result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); + result = prime * result + ((uploadLocation == null) ? 0 : uploadLocation.hashCode()); result = prime * result + ((uploadToken == null) ? 0 : uploadToken.hashCode()); result = prime * result + ((uploadUrl == null) ? 0 : uploadUrl.hashCode()); result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); @@ -218,6 +240,11 @@ public class UploadStore { return false; if (lastUploadedByte != other.lastUploadedByte) return false; + if (lastUploadedChunkTimestamp == null) { + if (other.lastUploadedChunkTimestamp != null) + return false; + } else if (!lastUploadedChunkTimestamp.equals(other.lastUploadedChunkTimestamp)) + return false; if (mediaType == null) { if (other.mediaType != null) return false; @@ -234,6 +261,11 @@ public class UploadStore { return false; } else if (!tokenExpirationTime.equals(other.tokenExpirationTime)) return false; + if (uploadLocation == null) { + if (other.uploadLocation != null) + return false; + } else if (!uploadLocation.equals(other.uploadLocation)) + return false; if (uploadToken == null) { if (other.uploadToken != null) return false; @@ -259,8 +291,9 @@ public class UploadStore { .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") - .append(retries).append(", partId=").append(partId).append(", lastUploadedByte=") - .append(lastUploadedByte).append("]").toString(); + .append(retries).append(", partId=").append(partId) + .append(", lastUploadedByte=").append(lastUploadedByte) + .append("]").toString(); } } diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java b/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java index e82580e..29a19eb 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java +++ b/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java @@ -35,14 +35,15 @@ public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadS } private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { - String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte, lastuploadedchunktimestamp, uploadlocation) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + ttl + ";"; return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), uploadStore.getStatus().toString(), uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), uploadStore.getUploadUrl(), uploadStore.getRetries(), uploadStore.getPartId(), - uploadStore.getLastUploadedByte()); + uploadStore.getLastUploadedByte(), uploadStore.getLastUploadedChunkTimestamp(), + uploadStore.getUploadLocation()); } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index 107f3e3..9ddefbd 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -77,7 +77,7 @@ public class FileUploadController { logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); return new ResponseEntity<>(new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), - uploadInfo.getLastUploadedByte()), HttpStatus.INTERNAL_SERVER_ERROR); + uploadInfo.getPartId()), HttpStatus.INTERNAL_SERVER_ERROR); } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java index 00fc32e..2afc695 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java @@ -3,6 +3,8 @@ */ package biz.nynja.content.upload.rest; +import java.time.Instant; + import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,14 +46,14 @@ public class FileUploadService { public UploadResponse writeDataChunk(UploadStore uploadInfo, byte[] dataChunk, int partId, boolean last) throws Exception { - String fileName = retrieveFileName(uploadInfo); - String fileLocation; - fileLocation = storageProvider.initialize(fileName); - logger.info("Created output stream for file {}", fileLocation); + if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // Upload hasn't started yet + uploadInfo = initializeUpload(uploadInfo); + } + + storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last); - storageProvider.write(dataChunk, fileLocation, partId, last); - logger.info("Chunk {} was successfully written to output stream {}.", partId, fileName); + logger.info("Chunk {} was successfully written to output stream {}.", partId, uploadInfo.getFileName()); uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); if (!uploadTokenService.storeToken(uploadInfo, true)) { // In this case data chunk was successfully appended to file (when LOCAL storage is used), but @@ -74,6 +76,16 @@ public class FileUploadService { uploadInfo.getPartId()); } + private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { + String fileName = constructFileName(uploadInfo.getFileName()); + uploadInfo.setFileName(fileName); + + String uploadLocation = storageProvider.initialize(fileName); + uploadInfo.setUploadLocation(uploadLocation); + + return uploadInfo; + } + public String finalizeUpload(UploadStore uploadInfo) { FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, @@ -84,17 +96,6 @@ public class FileUploadService { return dowlnoadUrl; } - private String retrieveFileName(UploadStore uploadInfo) { - String fileName; - if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // upload hasn't started yet - fileName = constructFileName(uploadInfo.getFileName()); - uploadInfo.setFileName(fileName); - } else { - fileName = uploadInfo.getFileName(); - } - return fileName; - } - private String constructFileName(String fileName) { String generatedPrefix = RandomStringUtils.randomAlphabetic(10); return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); @@ -103,6 +104,7 @@ public class FileUploadService { private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { uploadInfo.setPartId(partId); uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); + uploadInfo.setLastUploadedChunkTimestamp(Instant.now().toEpochMilli()); uploadInfo.setStatus(UploadStatus.IN_PROGRESS); return uploadInfo; } -- GitLab From 173ecec30ec156c9ed7150eabc992b82204d5272 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Tue, 22 Jan 2019 16:43:41 +0200 Subject: [PATCH 19/29] NY-6474: Set correct header for access token in unit tests Signed-off-by: Ralitsa Todorova --- .../nynja/content/grpc/services/ContentServiceImplTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java index b5dd68f..631d9ab 100644 --- a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java +++ b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java @@ -83,8 +83,9 @@ public class ContentServiceImplTests extends GrpcServerTestBase { ContentServiceGrpc.ContentServiceBlockingStub contentServiceBlockingStub = ContentServiceGrpc .newBlockingStub(Optional.ofNullable(channel).orElse(inProcChannel)); Metadata header = new Metadata(); - Metadata.Key key = Metadata.Key.of("accessToken", Metadata.ASCII_STRING_MARSHALLER); - header.put(key, + Metadata.Key key = Metadata.Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER); + header.put(key, "Bearer " + + "eyJraWQiOiIyMDE4MDYwOCIsInR5cCI6IkpXVCIsImFsZyI6IkVTMjU2In0.eyJzdWIiOiJPVE5sWW1Jek5UQXRPVFZoT1MwME0ySmxMV0kzTkRjdFpEVmhPREV5TW1aalpqWTIiLCJhdWQiOiJNVEl6TVRJejphcHBDbGFzczoxMjMzMzMzIiwic2NvcGUiOiJhY2Nlc3MiLCJyb2xlcyI6W10sImlzcyI6Imh0dHBzOi8vYXV0aC5ueW5qYS5iaXovIiwiZXhwIjoxNTQyMDM4NDQ2LCJpYXQiOjE1NDIwMzQ4NDZ9._NB7HrnLeLNlHMk8bvxqrLn01TX97Y0bzylsaGiA6aEFKedKo8QciLePiLPyEEFHllpGk1bSRnqdmJjGQvuX7A"); this.contentServiceBlockingStub = MetadataUtils.attachHeaders(contentServiceBlockingStub, header); return; -- GitLab From a12368add55a47a08a657940f5ea3980c7d8381a Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Fri, 25 Jan 2019 14:24:11 +0200 Subject: [PATCH 20/29] NY-6478 Delete endpoint - cancel endpoint; - exception flow; - cancel function impl; Signed-off-by: abotev-intracol --- .../content/file/storage/StorageProvider.java | 4 ++- .../storage/impl/GoogleStorageProvider.java | 36 +++++++++---------- .../storage/impl/LocalStorageProvider.java | 8 ++++- .../grpc/services/ContentServiceImpl.java | 17 +++++++-- .../upload/rest/FileUploadController.java | 26 ++++++++++++-- .../upload/rest/FileUploadService.java | 17 +++++++-- 6 files changed, 79 insertions(+), 29 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index c62c516..6264d3f 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -3,6 +3,8 @@ */ package biz.nynja.content.file.storage; +import java.io.IOException; + /** * @author Angel.Botev * @@ -13,7 +15,7 @@ public interface StorageProvider { public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws Exception; - public void close(String fileLocation); + public void close(String fileLocation) throws IOException; public String getFileUrl(String filename); diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index 0d971e7..6d0dc0a 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -56,7 +56,7 @@ public class GoogleStorageProvider implements StorageProvider { httpTransport = GoogleNetHttpTransport.newTrustedTransport(); // Build an account credential. GoogleCredential credential = GoogleCredential.getApplicationDefault(); - credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_READ_WRITE)); + credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)); requestFactory = httpTransport.createRequestFactory(credential); } catch (GeneralSecurityException | IOException e) { logger.error("Error with Google credentials: {}", e.getMessage()); @@ -82,7 +82,8 @@ public class GoogleStorageProvider implements StorageProvider { @Override public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws Exception { - logger.debug("Writing chunk: {} with length {} to: {}", chunkCount, data.length, fileLocation); + logger.debug("Writing partId: {} with length {} to: {}", chunkCount, data.length, fileLocation); + --chunkCount; // It's necessary because chunkCount starts from 1 not from 0 try (InputStream inputStream = new ByteBufInputStream(Unpooled.wrappedBuffer(data))) { int length = Math.min(Unpooled.wrappedBuffer(data).readableBytes(), storageConfiguration.getUploadChunkSize()); @@ -105,28 +106,23 @@ public class GoogleStorageProvider implements StorageProvider { } @Override - public void close(String fileLocation) { + public void close(String fileLocation) throws IOException { logger.debug("Cancel resumable Google upload for location: {}", fileLocation); String URI = fileLocation; GenericUrl url = new GenericUrl(URI); HttpRequest req; - try { - req = requestFactory.buildDeleteRequest(url); - HttpHeaders headers = new HttpHeaders(); - headers.setContentLength((long) 0); - req.setHeaders(headers); - logger.debug("Executing DELETE request to: {}", URI); - // Execute request - HttpResponse resp = req.execute(); - if (resp.getStatusCode() == 200) { - logger.debug("Successfull canceled Google upload for location: {}", fileLocation); - } else { - logger.error("Error canceling Google upload. Status code: {}, message: ", resp.getStatusCode(), - resp.getStatusMessage()); - } - } catch (IOException e) { - logger.error("Error canceling Google upload: {}", e.getMessage()); - logger.debug("Error canceling Google upload: {}", e.getCause()); + req = requestFactory.buildDeleteRequest(url); + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength((long) 0); + req.setHeaders(headers); + logger.debug("Executing DELETE request to: {}", URI); + // Execute request + HttpResponse resp = req.execute(); + if (resp.getStatusCode() == 200) { + logger.debug("Successfull canceled Google upload for location: {}", fileLocation); + } else { + logger.error("Error canceling Google upload. Status code: {}, message: ", resp.getStatusCode(), + resp.getStatusMessage()); } } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 8d4bcbb..9f4e24d 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -7,6 +7,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +56,11 @@ public class LocalStorageProvider implements StorageProvider { } @Override - public void close(String fileLocation) { + public void close(String fileLocation) throws IOException { + logger.debug("Cancel resumable upload for location: {}", fileLocation); + Files.delete(Paths.get(fileLocation)); + logger.debug("Successfull deleted file: {}", fileLocation); + logger.debug("Successfull canceled upload for location: {}", fileLocation); } private String constructFilePath(String contentStoreLocation, String fileName) { diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 8188c84..efc6978 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -5,6 +5,7 @@ package biz.nynja.content.grpc.services; import static biz.nynja.content.core.validation.Validators.util; +import java.io.IOException; import java.time.Instant; import java.util.Optional; import java.util.UUID; @@ -192,9 +193,9 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas return; } receivedBytesCount += data.length; + chunkCount++; storageProvider.write(data, fileLocation, chunkCount, (uploadInfo.getFileSize() == receivedBytesCount)); reachedSize += data.length; - chunkCount++; logger.info("Chunk {} was successfully written to output stream {}.", chunkCount, fileName); } catch (Exception e) { logger.error("Error writing data for file {}: {}", fileName, e.getMessage()); @@ -222,7 +223,12 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); - storageProvider.close(fileLocation); + try { + storageProvider.close(fileLocation); + } catch (IOException e) { + logger.error("Error closing upload job: {}", e.getMessage()); + logger.debug("Error closing upload job: {}", e.getCause()); + } uploadTokenService.deleteToken(uploadInfo.getUploadToken()); logger.info("File sucessfully uploaded."); } @@ -231,7 +237,12 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas uploadInfo.setRetries(uploadInfo.getRetries() + 1); uploadInfo.setStatus(UploadStatus.FAILED); uploadTokenService.storeToken(uploadInfo, false); - storageProvider.close(fileLocation); + try { + storageProvider.close(fileLocation); + } catch (IOException e) { + logger.error("Error closing upload job: {}", e.getMessage()); + logger.debug("Error closing upload job: {}", e.getCause()); + } } private String constructFileName(String fileName) { diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index 9ddefbd..657fe96 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -11,6 +11,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -76,9 +77,30 @@ public class FileUploadController { } catch (Exception e) { logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); - return new ResponseEntity<>(new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), - uploadInfo.getPartId()), HttpStatus.INTERNAL_SERVER_ERROR); + return new ResponseEntity<>( + new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), uploadInfo.getPartId()), + HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + @DeleteMapping("/{jobId}") + public ResponseEntity cancelUploading(@PathVariable String jobId, + @RequestParam("token") String uploadToken) { + + logger.info("Cancel request recived for jobId: {} and token: {}", jobId, uploadToken); + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (!uploadInfoResult.isPresent()) { + logger.error( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished."); + return new ResponseEntity<>( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished.", + HttpStatus.BAD_REQUEST); } + if(!fileUploadService.cancelUploadProcess(uploadInfoResult.get())) { + return new ResponseEntity<>("Error canceling upload job. File not found or not removed.", HttpStatus.INTERNAL_SERVER_ERROR); + } + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); } } diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java index 2afc695..6ad963e 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java @@ -3,6 +3,7 @@ */ package biz.nynja.content.upload.rest; +import java.io.IOException; import java.time.Instant; import org.apache.commons.lang3.RandomStringUtils; @@ -68,11 +69,11 @@ public class FileUploadService { } if (last) { finalizeUpload(uploadInfo); - return new UploadResponse("File uplaoad successfully completed.", UploadStatus.COMPLETED, + return new UploadResponse("File upload successfully completed.", UploadStatus.COMPLETED, uploadInfo.getPartId()); } - return new UploadResponse("Data chunk uplaoad successfully completed.", UploadStatus.IN_PROGRESS, + return new UploadResponse("Data chunk upload successfully completed.", UploadStatus.IN_PROGRESS, uploadInfo.getPartId()); } @@ -112,4 +113,16 @@ public class FileUploadService { private String constructDownloadUrl(String fileKey) { return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); } + + public boolean cancelUploadProcess(UploadStore uploadInfo) { + try { + this.storageProvider.close(uploadInfo.getUploadLocation()); + } catch (IOException e) { + logger.error("Error canceling upload job: {}", e.getMessage()); + logger.debug("Error canceling upload job: {}", e.getCause()); + return false; + } + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + return true; + } } -- GitLab From 5c4c3a3736f11869fc12097a406f5e12769afe23 Mon Sep 17 00:00:00 2001 From: Ralitsa Todorova Date: Mon, 28 Jan 2019 12:03:09 +0200 Subject: [PATCH 21/29] NY-6479: REST upload unit tests Signed-off-by: Ralitsa Todorova --- .../upload/rest/FileUploadController.java | 4 +- .../rest/validation/ChunkUploadValidator.java | 6 + .../file/upload/FileUploadControllerTest.java | 199 ++++++++++++++++++ .../biz/nynja/content/file/upload/Util.java | 72 +++++++ 4 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java create mode 100644 src/test/java/biz/nynja/content/file/upload/Util.java diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java index 9ddefbd..04350d6 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java @@ -5,8 +5,6 @@ package biz.nynja.content.upload.rest; import java.util.Optional; -import javax.servlet.http.HttpServletRequest; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -48,7 +46,7 @@ public class FileUploadController { @PutMapping("/{jobId}") public ResponseEntity uploadFile(@PathVariable String jobId, @RequestParam("token") String uploadToken, @RequestParam("partId") int partId, - @RequestParam("last") boolean last, HttpServletRequest request, @RequestBody byte[] dataChunk) { + @RequestParam("last") boolean last, @RequestBody byte[] dataChunk) { logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); UploadStore uploadInfo; diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java b/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java index 6e325e2..a044084 100644 --- a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java +++ b/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java @@ -68,6 +68,12 @@ public class ChunkUploadValidator { logger.error("Uploaded file size does not meet expectations - expected: {}, actual: {}.", expected, actual); return "Uploaded file size does not meet expectations."; } + if (!lastChunk && matchesExpectedSize(actual, expected)) { + logger.error( + "Uploaded file size so far reached expected overal file size, but this is not a final chunk - expected: {}, actual: {}.", + expected, actual); + return "Uploaded file size so far reached expected overal file size, but this is not a final chunk."; + } return null; } diff --git a/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java new file mode 100644 index 0000000..0963bde --- /dev/null +++ b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java @@ -0,0 +1,199 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload; + +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.Optional; + +import javax.annotation.Resource; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.web.context.WebApplicationContext; + +import biz.nynja.content.upload.models.UploadStatus; +import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.upload.rest.FileUploadController; +import biz.nynja.content.upload.rest.FileUploadService; +import biz.nynja.content.upload.rest.UploadResponse; +import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@RunWith(SpringRunner.class) +@WebMvcTest(FileUploadController.class) +@ContextConfiguration(classes = { FileUploadController.class, ChunkUploadValidator.class, Util.class }) +@ActiveProfiles("dev") +public class FileUploadControllerTest { + + @Resource + private WebApplicationContext wac; + + @MockBean + private FileUploadService fileUploadService; + + @MockBean + private UploadTokenService uploadTokenService; + + @Autowired + private MockMvc mockMvc; + + @Autowired + @Qualifier("uploadInfo") + private UploadStore uploadInfo; + + @Autowired + @Qualifier("uploadInfoInterm") + private UploadStore uploadInfoInterm; + + @Autowired + @Qualifier("uploadResponseFinalSuccess") + private UploadResponse uploadResponseFinalSuccess; + + @Autowired + @Qualifier("uploadResponseIntermediateChunkSuccess") + private UploadResponse uploadResponseIntermediateChunkSuccess; + + private final String UPLOAD_URL = "/file/upload/"; + + @Test + public void testUploadFinalChunkCreated() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + given(fileUploadService.writeDataChunk(uploadInfo, Util.DATA_CHUNK, 1, true)) + .willReturn(uploadResponseFinalSuccess); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, true)) + .content(Util.DATA_CHUNK)).andExpect(status().isCreated()); + } + + @Test + public void testUploadFinalChunkResponse() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + given(fileUploadService.writeDataChunk(uploadInfo, Util.DATA_CHUNK, 1, true)) + .willReturn(uploadResponseFinalSuccess); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, true)) + .content(Util.DATA_CHUNK)).andExpect(status().isCreated()) + .andExpect(jsonPath("$.message").value(uploadResponseFinalSuccess.getMessage())) + .andExpect(jsonPath("$.status").value(UploadStatus.COMPLETED.name())) + .andExpect(jsonPath("$.lastSuccessfulChunk").value(1)) + .andExpect(jsonPath("$.downloadLink").value(Util.DOWNLOAD_LINK)); + } + + @Test + public void testUploadIntermediateChunkOk() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfoInterm)); + given(fileUploadService.writeDataChunk(uploadInfoInterm, Util.DATA_CHUNK, 1, false)) + .willReturn(new UploadResponse("Success", UploadStatus.IN_PROGRESS, 1)); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isCreated()) + .andExpect(jsonPath("$.status").value(UploadStatus.IN_PROGRESS.name())) + .andExpect(jsonPath("$.lastSuccessfulChunk").value(1)).andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + @Test + public void testUploadIntermediateChunkReachedFileSize() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + given(fileUploadService.writeDataChunk(uploadInfo, Util.DATA_CHUNK, 1, false)) + .willReturn(new UploadResponse("Size not match.")); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + @Test + public void testUploadWrongDataChunk() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 2, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.status").value(UploadStatus.PENDING.name())) + .andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + @Test + public void testUploadEmptyBody() throws Exception { + + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false))) + .andExpect(status().isBadRequest()); + } + + @Test + public void testUploadInvalidJobId() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + mockMvc.perform(put(buildUploadRequest(Util.INVALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()); + } + + @Test + public void testUploadEmptyJobId() throws Exception { + + mockMvc.perform(put(buildUploadRequest("", Util.VALID_UPLOAD_TOKEN, 1, false)).content(Util.DATA_CHUNK)) + .andExpect(status().isNotFound()); + } + + @Test + public void testUploadEmptyToken() throws Exception { + + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, "", 1, false)).content(Util.DATA_CHUNK)) + .andExpect(status().isBadRequest()); + } + + @Test + public void testUploadEmptyLast() throws Exception { + + mockMvc.perform( + put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1)).content(Util.DATA_CHUNK)) + .andExpect(status().isBadRequest()); + } + + @Test + public void testUploadEmptyPartId() throws Exception { + + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()); + } + + @Test + public void testUploadInvalidUploadToken() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.INVALID_UPLOAD_TOKEN)).willReturn(Optional.empty()); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + private String buildUploadRequest(String jobId, String token, int partId, boolean last) { + return new StringBuilder(UPLOAD_URL).append(jobId).append("?token=").append(token).append("&partId=") + .append(partId).append("&last=").append(last).toString(); + } + + private String buildUploadRequest(String jobId, String token, boolean last) { + return new StringBuilder(UPLOAD_URL).append(jobId).append("?token=").append(token).append("&last=").append(last) + .toString(); + } + + private String buildUploadRequest(String jobId, String token, int partId) { + return new StringBuilder(UPLOAD_URL).append(jobId).append("?token=").append(token).append("&partId=") + .append(partId).toString(); + } +} diff --git a/src/test/java/biz/nynja/content/file/upload/Util.java b/src/test/java/biz/nynja/content/file/upload/Util.java new file mode 100644 index 0000000..9e9085a --- /dev/null +++ b/src/test/java/biz/nynja/content/file/upload/Util.java @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload; + +import java.util.Random; +import java.util.UUID; + +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; + +import biz.nynja.content.upload.models.UploadStatus; +import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.upload.rest.UploadResponse; + +/** + * @author Ralitsa Todorova + * + */ +@TestConfiguration +public class Util { + + public final static String VALID_UPLOAD_TOKEN = "qwertyuiopasdfghjklzxcvbnm"; + public final static String INVALID_UPLOAD_TOKEN = "qwertyuiopasdfghjklzxcvb"; + public final static String VALID_UPLOAD_JOBID = "3318f002-5fc1-4aff-bf70-9dbf728c29ca"; + public final static String INVALID_UPLOAD_JOBID = "3318f002-5fc1-4aff-bf70-9d"; + public final static int FILE_SIZE = 12345; + public final static String FILE_NAME = "somefilename.txt"; + public final static byte[] DATA_CHUNK = generateBytes(); + public final static String DOWNLOAD_LINK = "https://content.nynja.biz/file/download/kkk"; + + @Bean + public UploadStore uploadInfo() { + UploadStore uploadInfo = new UploadStore(); + uploadInfo.setStatus(UploadStatus.PENDING); + uploadInfo.setUploadToken(VALID_UPLOAD_TOKEN); + uploadInfo.setJobId(UUID.fromString(VALID_UPLOAD_JOBID)); + uploadInfo.setFileSize(FILE_SIZE); + uploadInfo.setFileName(FILE_NAME); + uploadInfo.setLastUploadedByte(0); + return uploadInfo; + } + + @Bean + public UploadStore uploadInfoInterm() { + UploadStore uploadInfo = new UploadStore(); + uploadInfo.setStatus(UploadStatus.PENDING); + uploadInfo.setUploadToken(VALID_UPLOAD_TOKEN); + uploadInfo.setJobId(UUID.fromString(VALID_UPLOAD_JOBID)); + uploadInfo.setFileSize(FILE_SIZE * 2); + uploadInfo.setFileName(FILE_NAME); + uploadInfo.setLastUploadedByte(0); + return uploadInfo; + } + @Bean + public UploadResponse uploadResponseIntermediateChunkSuccess() { + UploadResponse uploadResponse = new UploadResponse("Upload success.", UploadStatus.IN_PROGRESS, 1, null); + return uploadResponse; + } + + @Bean + public UploadResponse uploadResponseFinalSuccess() { + UploadResponse uploadResponse = new UploadResponse("Upload success.", UploadStatus.COMPLETED, 1, DOWNLOAD_LINK); + return uploadResponse; + } + + private static byte[] generateBytes() { + byte[] b = new byte[FILE_SIZE]; + new Random().nextBytes(b); + return b; + } +} -- GitLab From bc53bd8ec9308409ef32ec058f3488cbcca375a5 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Mon, 28 Jan 2019 13:53:20 +0200 Subject: [PATCH 22/29] NY-6478 Delete endpoint - fix comments; Signed-off-by: abotev-intracol --- .../nynja/content/file/storage/impl/LocalStorageProvider.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index 9f4e24d..29c75aa 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -57,9 +57,8 @@ public class LocalStorageProvider implements StorageProvider { @Override public void close(String fileLocation) throws IOException { - logger.debug("Cancel resumable upload for location: {}", fileLocation); + logger.debug("Canceling resumable upload for location: {}", fileLocation); Files.delete(Paths.get(fileLocation)); - logger.debug("Successfull deleted file: {}", fileLocation); logger.debug("Successfull canceled upload for location: {}", fileLocation); } -- GitLab From 5118973853d7ecf64caf8a6abf7c3e10958ab986 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 30 Jan 2019 12:24:23 +0200 Subject: [PATCH 23/29] NY-6838 Move rest upload package to file->upload package - move package; - fix imports; - fix in ExceptionHandler; Signed-off-by: abotev-intracol --- ...a => FileControllersExceptionHandler.java} | 16 +- .../file/metadata/FileMetadataService.java | 2 +- .../upload/models/UploadStatus.java | 26 +- .../{ => file}/upload/models/UploadStore.java | 598 +++++++++--------- .../CustomizedUploadStoreRepository.java | 28 +- .../CustomizedUploadStoreRepositoryImpl.java | 98 +-- .../repositories/UploadStoreRepository.java | 40 +- .../upload/rest/FileUploadController.java | 208 +++--- .../upload/rest/FileUploadService.java | 256 ++++---- .../upload/rest/UploadResponse.java | 224 +++---- .../rest/validation/ChunkUploadValidator.java | 160 ++--- .../rest/validation/ValidationResult.java | 52 +- .../{ => file}/upload/token/UploadToken.java | 248 ++++---- .../token/UploadTokenResponseProvider.java | 68 +- .../upload/token/UploadTokenService.java | 202 +++--- .../upload/token/UploadTokenValidator.java | 254 ++++---- .../MediaTypesConfiguration.java | 2 +- .../UploadTokenConfiguration.java | 102 +-- .../grpc/interceptors/UploadInterceptor.java | 6 +- .../grpc/services/ContentServiceImpl.java | 14 +- .../download/FileDownloadControllerTest.java | 4 +- .../file/upload/FileUploadControllerTest.java | 14 +- .../biz/nynja/content/file/upload/Util.java | 6 +- .../services/ContentServiceImplTests.java | 6 +- .../upload/token/UploadTokenTests.java | 6 +- 25 files changed, 1322 insertions(+), 1318 deletions(-) rename src/main/java/biz/nynja/content/file/{download/FileControllerExceptionHandler.java => FileControllersExceptionHandler.java} (92%) rename src/main/java/biz/nynja/content/{ => file}/upload/models/UploadStatus.java (80%) rename src/main/java/biz/nynja/content/{ => file}/upload/models/UploadStore.java (96%) rename src/main/java/biz/nynja/content/{ => file}/upload/repositories/CustomizedUploadStoreRepository.java (81%) rename src/main/java/biz/nynja/content/{ => file}/upload/repositories/CustomizedUploadStoreRepositoryImpl.java (90%) rename src/main/java/biz/nynja/content/{ => file}/upload/repositories/UploadStoreRepository.java (80%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/FileUploadController.java (92%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/FileUploadService.java (94%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/UploadResponse.java (93%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/validation/ChunkUploadValidator.java (94%) rename src/main/java/biz/nynja/content/{ => file}/upload/rest/validation/ValidationResult.java (85%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/UploadToken.java (95%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/UploadTokenResponseProvider.java (94%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/UploadTokenService.java (91%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/UploadTokenValidator.java (94%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/configuration/MediaTypesConfiguration.java (96%) rename src/main/java/biz/nynja/content/{ => file}/upload/token/configuration/UploadTokenConfiguration.java (92%) diff --git a/src/main/java/biz/nynja/content/file/download/FileControllerExceptionHandler.java b/src/main/java/biz/nynja/content/file/FileControllersExceptionHandler.java similarity index 92% rename from src/main/java/biz/nynja/content/file/download/FileControllerExceptionHandler.java rename to src/main/java/biz/nynja/content/file/FileControllersExceptionHandler.java index 9139882..bd1381c 100644 --- a/src/main/java/biz/nynja/content/file/download/FileControllerExceptionHandler.java +++ b/src/main/java/biz/nynja/content/file/FileControllersExceptionHandler.java @@ -1,7 +1,7 @@ /** * Copyright (C) 2018 Nynja Inc. All rights reserved. */ -package biz.nynja.content.file.download; +package biz.nynja.content.file; import java.io.FileNotFoundException; import java.util.MissingResourceException; @@ -25,9 +25,9 @@ import org.springframework.web.client.HttpServerErrorException; import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; @ControllerAdvice -public class FileControllerExceptionHandler { +public class FileControllersExceptionHandler { - private static final Logger logger = LoggerFactory.getLogger(FileControllerExceptionHandler.class); + private static final Logger logger = LoggerFactory.getLogger(FileControllersExceptionHandler.class); @ExceptionHandler({ IllegalArgumentException.class, MethodArgumentNotValidException.class, HttpMessageNotReadableException.class, ServletRequestBindingException.class }) @@ -58,14 +58,14 @@ public class FileControllerExceptionHandler { public static ResponseEntity handleNotFoundExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_FOUND, e), HttpStatus.NOT_FOUND); + return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_FOUND, e.getMessage()), HttpStatus.NOT_FOUND); } @ExceptionHandler({ HttpMediaTypeNotAcceptableException.class }) public ResponseEntity handleNotAcceptableExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_ACCEPTABLE, e), + return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_ACCEPTABLE, e.getMessage()), HttpStatus.NOT_ACCEPTABLE); } @@ -73,20 +73,20 @@ public class FileControllerExceptionHandler { public ResponseEntity handleUnsupportedMediaTypeExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.UNSUPPORTED_MEDIA_TYPE, e), + return new ResponseEntity(new ExceptionBody(HttpStatus.UNSUPPORTED_MEDIA_TYPE, e.getMessage()), HttpStatus.UNSUPPORTED_MEDIA_TYPE); } @ExceptionHandler({ MethodArgumentTypeMismatchException.class }) public ResponseEntity handleException(Exception e) { - return new ResponseEntity(new ExceptionBody(HttpStatus.BAD_REQUEST, e), HttpStatus.BAD_REQUEST); + return new ResponseEntity(new ExceptionBody(HttpStatus.BAD_REQUEST, e.getMessage()), HttpStatus.BAD_REQUEST); } @ExceptionHandler({ HttpServerErrorException.class, InternalError.class, Exception.class }) public ResponseEntity handleInternalServerExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.INTERNAL_SERVER_ERROR, e), + return new ResponseEntity(new ExceptionBody(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage()), HttpStatus.INTERNAL_SERVER_ERROR); } diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index 427f71f..7c885bd 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.metadata.repositories.FileMetadataReposiory; import biz.nynja.content.file.storage.Provider; -import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.file.upload.models.UploadStore; /** * @author Ralitsa Todorova diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java b/src/main/java/biz/nynja/content/file/upload/models/UploadStatus.java similarity index 80% rename from src/main/java/biz/nynja/content/upload/models/UploadStatus.java rename to src/main/java/biz/nynja/content/file/upload/models/UploadStatus.java index ac9e5e9..bcb595b 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStatus.java +++ b/src/main/java/biz/nynja/content/file/upload/models/UploadStatus.java @@ -1,13 +1,13 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.models; - -/** - * @author Ralitsa Todorova - * - */ -public enum UploadStatus { - - PENDING, IN_PROGRESS, FAILED, COMPLETED; -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.models; + +/** + * @author Ralitsa Todorova + * + */ +public enum UploadStatus { + + PENDING, IN_PROGRESS, FAILED, COMPLETED; +} diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java similarity index 96% rename from src/main/java/biz/nynja/content/upload/models/UploadStore.java rename to src/main/java/biz/nynja/content/file/upload/models/UploadStore.java index 66f4757..b902848 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java @@ -1,299 +1,299 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.models; - -import java.util.UUID; - -import org.springframework.data.cassandra.core.mapping.PrimaryKey; -import org.springframework.data.cassandra.core.mapping.Table; - -/** - * @author Ralitsa Todorova - * - */ -@Table -public class UploadStore { - - @PrimaryKey - private String uploadToken; - private Long tokenExpirationTime; - private String uploadUrl; - private UUID jobId; - private String deviceId; - private UUID accountId; - private String fileName; - private int fileSize; - private String mediaType; - private String uploadedFrom; - private UploadStatus status; - private int retries; - private int partId; - private int lastUploadedByte; - private String uploadLocation; - private Long lastUploadedChunkTimestamp; - - public UploadStore() { - } - - public UploadStore(String uploadToken, Long tokenExpirationTime, String uploadUrl, UUID jobId, String deviceId, - UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, UploadStatus status) { - this.uploadToken = uploadToken; - this.tokenExpirationTime = tokenExpirationTime; - this.uploadUrl = uploadUrl; - this.jobId = jobId; - this.deviceId = deviceId; - this.accountId = accountId; - this.fileName = fileName; - this.fileSize = fileSize; - this.mediaType = mediaType; - this.uploadedFrom = uploadedFrom; - this.status = status; - this.retries = 0; - this.partId = 0; - this.lastUploadedByte = 0; - this.uploadLocation = ""; - this.lastUploadedChunkTimestamp = 0L; - } - - public String getUploadToken() { - return uploadToken; - } - - public String getUploadUrl() { - return uploadUrl; - } - - public void setUploadUrl(String uploadUrl) { - this.uploadUrl = uploadUrl; - } - - public UUID getJobId() { - return jobId; - } - - public void setJobId(UUID jobId) { - this.jobId = jobId; - } - - public String getDeviceId() { - return deviceId; - } - - public void setDeviceId(String deviceId) { - this.deviceId = deviceId; - } - - public UUID getAccountId() { - return accountId; - } - - public void setAccountId(UUID accountId) { - this.accountId = accountId; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public int getFileSize() { - return fileSize; - } - - public void setFileSize(int fileSize) { - this.fileSize = fileSize; - } - - public String getMediaType() { - return mediaType; - } - - public void setMediaType(String mediaType) { - this.mediaType = mediaType; - } - - public String getUploadedFrom() { - return uploadedFrom; - } - - public void setUploadedFrom(String uploadedFrom) { - this.uploadedFrom = uploadedFrom; - } - - public UploadStatus getStatus() { - return status; - } - - public void setStatus(UploadStatus status) { - this.status = status; - } - - public void setUploadToken(String uploadToken) { - this.uploadToken = uploadToken; - } - - public Long getTokenExpirationTime() { - return tokenExpirationTime; - } - - public void setTokenExpirationTime(Long tokenExpirationTime) { - this.tokenExpirationTime = tokenExpirationTime; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - this.retries = retries; - } - - public int getPartId() { - return partId; - } - - public void setPartId(int partId) { - this.partId = partId; - } - - public int getLastUploadedByte() { - return lastUploadedByte; - } - - public void setLastUploadedByte(int lastUploadedByte) { - this.lastUploadedByte = lastUploadedByte; - } - - public String getUploadLocation() { - return uploadLocation; - } - - public void setUploadLocation(String uploadLocation) { - this.uploadLocation = uploadLocation; - } - - public Long getLastUploadedChunkTimestamp() { - return lastUploadedChunkTimestamp; - } - - public void setLastUploadedChunkTimestamp(Long lastUploadedChunkTimestamp) { - this.lastUploadedChunkTimestamp = lastUploadedChunkTimestamp; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((accountId == null) ? 0 : accountId.hashCode()); - result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); - result = prime * result + ((fileName == null) ? 0 : fileName.hashCode()); - result = prime * result + fileSize; - result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); - result = prime * result + lastUploadedByte; - result = prime * result + ((lastUploadedChunkTimestamp == null) ? 0 : lastUploadedChunkTimestamp.hashCode()); - result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); - result = prime * result + partId; - result = prime * result + retries; - result = prime * result + ((status == null) ? 0 : status.hashCode()); - result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); - result = prime * result + ((uploadLocation == null) ? 0 : uploadLocation.hashCode()); - result = prime * result + ((uploadToken == null) ? 0 : uploadToken.hashCode()); - result = prime * result + ((uploadUrl == null) ? 0 : uploadUrl.hashCode()); - result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - UploadStore other = (UploadStore) obj; - if (accountId == null) { - if (other.accountId != null) - return false; - } else if (!accountId.equals(other.accountId)) - return false; - if (deviceId == null) { - if (other.deviceId != null) - return false; - } else if (!deviceId.equals(other.deviceId)) - return false; - if (fileName == null) { - if (other.fileName != null) - return false; - } else if (!fileName.equals(other.fileName)) - return false; - if (fileSize != other.fileSize) - return false; - if (jobId == null) { - if (other.jobId != null) - return false; - } else if (!jobId.equals(other.jobId)) - return false; - if (lastUploadedByte != other.lastUploadedByte) - return false; - if (lastUploadedChunkTimestamp == null) { - if (other.lastUploadedChunkTimestamp != null) - return false; - } else if (!lastUploadedChunkTimestamp.equals(other.lastUploadedChunkTimestamp)) - return false; - if (mediaType == null) { - if (other.mediaType != null) - return false; - } else if (!mediaType.equals(other.mediaType)) - return false; - if (partId != other.partId) - return false; - if (retries != other.retries) - return false; - if (status != other.status) - return false; - if (tokenExpirationTime == null) { - if (other.tokenExpirationTime != null) - return false; - } else if (!tokenExpirationTime.equals(other.tokenExpirationTime)) - return false; - if (uploadLocation == null) { - if (other.uploadLocation != null) - return false; - } else if (!uploadLocation.equals(other.uploadLocation)) - return false; - if (uploadToken == null) { - if (other.uploadToken != null) - return false; - } else if (!uploadToken.equals(other.uploadToken)) - return false; - if (uploadUrl == null) { - if (other.uploadUrl != null) - return false; - } else if (!uploadUrl.equals(other.uploadUrl)) - return false; - if (uploadedFrom == null) { - if (other.uploadedFrom != null) - return false; - } else if (!uploadedFrom.equals(other.uploadedFrom)) - return false; - return true; - } - - @Override - public String toString() { - return new StringBuilder("UploadStore [uploadToken=").append(uploadToken).append(", tokenExpirationTime=") - .append(tokenExpirationTime).append(", uploadUrl=").append(uploadUrl).append(", jobId=").append(jobId) - .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") - .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) - .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") - .append(retries).append(", partId=").append(partId) - .append(", lastUploadedByte=").append(lastUploadedByte) - .append("]").toString(); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.models; + +import java.util.UUID; + +import org.springframework.data.cassandra.core.mapping.PrimaryKey; +import org.springframework.data.cassandra.core.mapping.Table; + +/** + * @author Ralitsa Todorova + * + */ +@Table +public class UploadStore { + + @PrimaryKey + private String uploadToken; + private Long tokenExpirationTime; + private String uploadUrl; + private UUID jobId; + private String deviceId; + private UUID accountId; + private String fileName; + private int fileSize; + private String mediaType; + private String uploadedFrom; + private UploadStatus status; + private int retries; + private int partId; + private int lastUploadedByte; + private String uploadLocation; + private Long lastUploadedChunkTimestamp; + + public UploadStore() { + } + + public UploadStore(String uploadToken, Long tokenExpirationTime, String uploadUrl, UUID jobId, String deviceId, + UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, UploadStatus status) { + this.uploadToken = uploadToken; + this.tokenExpirationTime = tokenExpirationTime; + this.uploadUrl = uploadUrl; + this.jobId = jobId; + this.deviceId = deviceId; + this.accountId = accountId; + this.fileName = fileName; + this.fileSize = fileSize; + this.mediaType = mediaType; + this.uploadedFrom = uploadedFrom; + this.status = status; + this.retries = 0; + this.partId = 0; + this.lastUploadedByte = 0; + this.uploadLocation = ""; + this.lastUploadedChunkTimestamp = 0L; + } + + public String getUploadToken() { + return uploadToken; + } + + public String getUploadUrl() { + return uploadUrl; + } + + public void setUploadUrl(String uploadUrl) { + this.uploadUrl = uploadUrl; + } + + public UUID getJobId() { + return jobId; + } + + public void setJobId(UUID jobId) { + this.jobId = jobId; + } + + public String getDeviceId() { + return deviceId; + } + + public void setDeviceId(String deviceId) { + this.deviceId = deviceId; + } + + public UUID getAccountId() { + return accountId; + } + + public void setAccountId(UUID accountId) { + this.accountId = accountId; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public int getFileSize() { + return fileSize; + } + + public void setFileSize(int fileSize) { + this.fileSize = fileSize; + } + + public String getMediaType() { + return mediaType; + } + + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + public String getUploadedFrom() { + return uploadedFrom; + } + + public void setUploadedFrom(String uploadedFrom) { + this.uploadedFrom = uploadedFrom; + } + + public UploadStatus getStatus() { + return status; + } + + public void setStatus(UploadStatus status) { + this.status = status; + } + + public void setUploadToken(String uploadToken) { + this.uploadToken = uploadToken; + } + + public Long getTokenExpirationTime() { + return tokenExpirationTime; + } + + public void setTokenExpirationTime(Long tokenExpirationTime) { + this.tokenExpirationTime = tokenExpirationTime; + } + + public int getRetries() { + return retries; + } + + public void setRetries(int retries) { + this.retries = retries; + } + + public int getPartId() { + return partId; + } + + public void setPartId(int partId) { + this.partId = partId; + } + + public int getLastUploadedByte() { + return lastUploadedByte; + } + + public void setLastUploadedByte(int lastUploadedByte) { + this.lastUploadedByte = lastUploadedByte; + } + + public String getUploadLocation() { + return uploadLocation; + } + + public void setUploadLocation(String uploadLocation) { + this.uploadLocation = uploadLocation; + } + + public Long getLastUploadedChunkTimestamp() { + return lastUploadedChunkTimestamp; + } + + public void setLastUploadedChunkTimestamp(Long lastUploadedChunkTimestamp) { + this.lastUploadedChunkTimestamp = lastUploadedChunkTimestamp; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((accountId == null) ? 0 : accountId.hashCode()); + result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); + result = prime * result + ((fileName == null) ? 0 : fileName.hashCode()); + result = prime * result + fileSize; + result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); + result = prime * result + lastUploadedByte; + result = prime * result + ((lastUploadedChunkTimestamp == null) ? 0 : lastUploadedChunkTimestamp.hashCode()); + result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); + result = prime * result + partId; + result = prime * result + retries; + result = prime * result + ((status == null) ? 0 : status.hashCode()); + result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); + result = prime * result + ((uploadLocation == null) ? 0 : uploadLocation.hashCode()); + result = prime * result + ((uploadToken == null) ? 0 : uploadToken.hashCode()); + result = prime * result + ((uploadUrl == null) ? 0 : uploadUrl.hashCode()); + result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadStore other = (UploadStore) obj; + if (accountId == null) { + if (other.accountId != null) + return false; + } else if (!accountId.equals(other.accountId)) + return false; + if (deviceId == null) { + if (other.deviceId != null) + return false; + } else if (!deviceId.equals(other.deviceId)) + return false; + if (fileName == null) { + if (other.fileName != null) + return false; + } else if (!fileName.equals(other.fileName)) + return false; + if (fileSize != other.fileSize) + return false; + if (jobId == null) { + if (other.jobId != null) + return false; + } else if (!jobId.equals(other.jobId)) + return false; + if (lastUploadedByte != other.lastUploadedByte) + return false; + if (lastUploadedChunkTimestamp == null) { + if (other.lastUploadedChunkTimestamp != null) + return false; + } else if (!lastUploadedChunkTimestamp.equals(other.lastUploadedChunkTimestamp)) + return false; + if (mediaType == null) { + if (other.mediaType != null) + return false; + } else if (!mediaType.equals(other.mediaType)) + return false; + if (partId != other.partId) + return false; + if (retries != other.retries) + return false; + if (status != other.status) + return false; + if (tokenExpirationTime == null) { + if (other.tokenExpirationTime != null) + return false; + } else if (!tokenExpirationTime.equals(other.tokenExpirationTime)) + return false; + if (uploadLocation == null) { + if (other.uploadLocation != null) + return false; + } else if (!uploadLocation.equals(other.uploadLocation)) + return false; + if (uploadToken == null) { + if (other.uploadToken != null) + return false; + } else if (!uploadToken.equals(other.uploadToken)) + return false; + if (uploadUrl == null) { + if (other.uploadUrl != null) + return false; + } else if (!uploadUrl.equals(other.uploadUrl)) + return false; + if (uploadedFrom == null) { + if (other.uploadedFrom != null) + return false; + } else if (!uploadedFrom.equals(other.uploadedFrom)) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadStore [uploadToken=").append(uploadToken).append(", tokenExpirationTime=") + .append(tokenExpirationTime).append(", uploadUrl=").append(uploadUrl).append(", jobId=").append(jobId) + .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") + .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) + .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") + .append(retries).append(", partId=").append(partId) + .append(", lastUploadedByte=").append(lastUploadedByte) + .append("]").toString(); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepository.java b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepository.java similarity index 81% rename from src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepository.java rename to src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepository.java index 793c38d..aae1f91 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepository.java +++ b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepository.java @@ -1,14 +1,14 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.repositories; - -/** - * @author Ralitsa Todorova - * @param - * - */ -public interface CustomizedUploadStoreRepository { - - S saveWithTtl(S uploadStore, int ttl); -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.repositories; + +/** + * @author Ralitsa Todorova + * @param + * + */ +public interface CustomizedUploadStoreRepository { + + S saveWithTtl(S uploadStore, int ttl); +} diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepositoryImpl.java similarity index 90% rename from src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java rename to src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepositoryImpl.java index 29a19eb..cd9c5b5 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java +++ b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepositoryImpl.java @@ -1,49 +1,49 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.repositories; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.cassandra.core.CassandraTemplate; - -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; - -/** - * @author Ralitsa Todorova - * - */ -public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadStoreRepository { - - private final CassandraTemplate cassandraTemplate; - private UploadTokenConfiguration uploadTokenConfiguration; - - @Autowired - public CustomizedUploadStoreRepositoryImpl(CassandraTemplate cassandraTemplate, - UploadTokenConfiguration uploadTokenConfiguration) { - this.cassandraTemplate = cassandraTemplate; - this.uploadTokenConfiguration = uploadTokenConfiguration; - } - - public S saveWithTtl(S uploadStore, int ttl) { - - if (!executeSaveWithTtl((UploadStore) uploadStore, uploadTokenConfiguration.getTokenTimeToLive())) { - throw new RuntimeException("Error inserting new upload info record."); - } else { - return uploadStore; - } - } - - private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { - String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte, lastuploadedchunktimestamp, uploadlocation) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " - + ttl + ";"; - return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), - uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), - uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), - uploadStore.getStatus().toString(), uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), - uploadStore.getUploadUrl(), uploadStore.getRetries(), uploadStore.getPartId(), - uploadStore.getLastUploadedByte(), uploadStore.getLastUploadedChunkTimestamp(), - uploadStore.getUploadLocation()); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.repositories; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.cassandra.core.CassandraTemplate; + +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; + +/** + * @author Ralitsa Todorova + * + */ +public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadStoreRepository { + + private final CassandraTemplate cassandraTemplate; + private UploadTokenConfiguration uploadTokenConfiguration; + + @Autowired + public CustomizedUploadStoreRepositoryImpl(CassandraTemplate cassandraTemplate, + UploadTokenConfiguration uploadTokenConfiguration) { + this.cassandraTemplate = cassandraTemplate; + this.uploadTokenConfiguration = uploadTokenConfiguration; + } + + public S saveWithTtl(S uploadStore, int ttl) { + + if (!executeSaveWithTtl((UploadStore) uploadStore, uploadTokenConfiguration.getTokenTimeToLive())) { + throw new RuntimeException("Error inserting new upload info record."); + } else { + return uploadStore; + } + } + + private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { + String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte, lastuploadedchunktimestamp, uploadlocation) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + + ttl + ";"; + return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), + uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), + uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), + uploadStore.getStatus().toString(), uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), + uploadStore.getUploadUrl(), uploadStore.getRetries(), uploadStore.getPartId(), + uploadStore.getLastUploadedByte(), uploadStore.getLastUploadedChunkTimestamp(), + uploadStore.getUploadLocation()); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/repositories/UploadStoreRepository.java b/src/main/java/biz/nynja/content/file/upload/repositories/UploadStoreRepository.java similarity index 80% rename from src/main/java/biz/nynja/content/upload/repositories/UploadStoreRepository.java rename to src/main/java/biz/nynja/content/file/upload/repositories/UploadStoreRepository.java index 9a78df4..bb5cef7 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/UploadStoreRepository.java +++ b/src/main/java/biz/nynja/content/file/upload/repositories/UploadStoreRepository.java @@ -1,20 +1,20 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.repositories; - -import org.springframework.data.cassandra.repository.CassandraRepository; -import org.springframework.stereotype.Repository; - -import biz.nynja.content.upload.models.UploadStore; - -/** - * @author Ralitsa Todorova - * - */ -@Repository -public interface UploadStoreRepository - extends CassandraRepository, CustomizedUploadStoreRepository { - - UploadStore findByUploadToken(String uploadToken); -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.repositories; + +import org.springframework.data.cassandra.repository.CassandraRepository; +import org.springframework.stereotype.Repository; + +import biz.nynja.content.file.upload.models.UploadStore; + +/** + * @author Ralitsa Todorova + * + */ +@Repository +public interface UploadStoreRepository + extends CassandraRepository, CustomizedUploadStoreRepository { + + UploadStore findByUploadToken(String uploadToken); +} diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java similarity index 92% rename from src/main/java/biz/nynja/content/upload/rest/FileUploadController.java rename to src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java index 0ba0e8b..759fd82 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadController.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java @@ -1,104 +1,104 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest; - -import java.util.Optional; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; -import biz.nynja.content.upload.rest.validation.ValidationResult; -import biz.nynja.content.upload.token.UploadTokenService; - -/** - * @author Ralitsa Todorova - * - */ -@RestController -@RequestMapping("/file/upload") -public class FileUploadController { - - private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); - - private UploadTokenService uploadTokenService; - private ChunkUploadValidator validator; - private FileUploadService fileUploadService; - - public FileUploadController(UploadTokenService uploadTokenService, ChunkUploadValidator validator, - FileUploadService fileUploadService) { - - this.uploadTokenService = uploadTokenService; - this.validator = validator; - this.fileUploadService = fileUploadService; - } - - @PutMapping("/{jobId}") - public ResponseEntity uploadFile(@PathVariable String jobId, - @RequestParam("token") String uploadToken, @RequestParam("partId") int partId, - @RequestParam("last") boolean last, @RequestBody byte[] dataChunk) { - - logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); - UploadStore uploadInfo; - Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); - if (uploadInfoResult.isPresent()) { - uploadInfo = uploadInfoResult.get(); - } else { - logger.error("You are not allowed to upload data. Provided token is either wrong or missing."); - return new ResponseEntity<>( - new UploadResponse( - "You are not allowed to upload data. Provided token is either wrong or missing."), - HttpStatus.BAD_REQUEST); - } - - Optional requestValidation = validator.validateUploadRequest(uploadInfo, dataChunk, jobId, - uploadToken, partId, last); - if (requestValidation.isPresent()) { - return new ResponseEntity<>(new UploadResponse(requestValidation.get().getMessage(), uploadInfo.getStatus(), - uploadInfo.getPartId()), HttpStatus.BAD_REQUEST); - } - - try { - UploadResponse result = fileUploadService.writeDataChunk(uploadInfo, dataChunk, partId, last); - return new ResponseEntity<>(result, HttpStatus.CREATED); - } catch (Exception e) { - logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); - logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); - return new ResponseEntity<>( - new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), uploadInfo.getPartId()), - HttpStatus.INTERNAL_SERVER_ERROR); - } - } - - @DeleteMapping("/{jobId}") - public ResponseEntity cancelUploading(@PathVariable String jobId, - @RequestParam("token") String uploadToken) { - - logger.info("Cancel request recived for jobId: {} and token: {}", jobId, uploadToken); - Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); - if (!uploadInfoResult.isPresent()) { - logger.error( - "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished."); - return new ResponseEntity<>( - "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished.", - HttpStatus.BAD_REQUEST); - } - if(!fileUploadService.cancelUploadProcess(uploadInfoResult.get())) { - return new ResponseEntity<>("Error canceling upload job. File not found or not removed.", HttpStatus.INTERNAL_SERVER_ERROR); - } - - return new ResponseEntity<>(HttpStatus.NO_CONTENT); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.file.upload.rest.validation.ValidationResult; +import biz.nynja.content.file.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@RestController +@RequestMapping("/file/upload") +public class FileUploadController { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); + + private UploadTokenService uploadTokenService; + private ChunkUploadValidator validator; + private FileUploadService fileUploadService; + + public FileUploadController(UploadTokenService uploadTokenService, ChunkUploadValidator validator, + FileUploadService fileUploadService) { + + this.uploadTokenService = uploadTokenService; + this.validator = validator; + this.fileUploadService = fileUploadService; + } + + @PutMapping("/{jobId}") + public ResponseEntity uploadFile(@PathVariable String jobId, + @RequestParam("token") String uploadToken, @RequestParam("partId") int partId, + @RequestParam("last") boolean last, @RequestBody byte[] dataChunk) { + + logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); + UploadStore uploadInfo; + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (uploadInfoResult.isPresent()) { + uploadInfo = uploadInfoResult.get(); + } else { + logger.error("You are not allowed to upload data. Provided token is either wrong or missing."); + return new ResponseEntity<>( + new UploadResponse( + "You are not allowed to upload data. Provided token is either wrong or missing."), + HttpStatus.BAD_REQUEST); + } + + Optional requestValidation = validator.validateUploadRequest(uploadInfo, dataChunk, jobId, + uploadToken, partId, last); + if (requestValidation.isPresent()) { + return new ResponseEntity<>(new UploadResponse(requestValidation.get().getMessage(), uploadInfo.getStatus(), + uploadInfo.getPartId()), HttpStatus.BAD_REQUEST); + } + + try { + UploadResponse result = fileUploadService.writeDataChunk(uploadInfo, dataChunk, partId, last); + return new ResponseEntity<>(result, HttpStatus.CREATED); + } catch (Exception e) { + logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); + logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); + return new ResponseEntity<>( + new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), uploadInfo.getPartId()), + HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + @DeleteMapping("/{jobId}") + public ResponseEntity cancelUploading(@PathVariable String jobId, + @RequestParam("token") String uploadToken) { + + logger.info("Cancel request recived for jobId: {} and token: {}", jobId, uploadToken); + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (!uploadInfoResult.isPresent()) { + logger.error( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished."); + return new ResponseEntity<>( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished.", + HttpStatus.BAD_REQUEST); + } + if(!fileUploadService.cancelUploadProcess(uploadInfoResult.get())) { + return new ResponseEntity<>("Error canceling upload job. File not found or not removed.", HttpStatus.INTERNAL_SERVER_ERROR); + } + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/rest/FileUploadService.java rename to src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index 6ad963e..1a235ef 100644 --- a/src/main/java/biz/nynja/content/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -1,128 +1,128 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest; - -import java.io.IOException; -import java.time.Instant; - -import org.apache.commons.lang3.RandomStringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import biz.nynja.content.file.metadata.FileMetadataService; -import biz.nynja.content.file.metadata.dto.FileMetadata; -import biz.nynja.content.file.storage.StorageProvider; -import biz.nynja.content.file.storage.StorageProviderPool; -import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; -import biz.nynja.content.upload.models.UploadStatus; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadTokenService; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class FileUploadService { - - private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); - - private StorageProvider storageProvider; - private UploadTokenService uploadTokenService; - private FileMetadataService fileMetadataService; - private ContentServiceConfiguration contentServiceConfiguration; - - @Autowired - public FileUploadService(StorageProviderPool storageProviderPool, UploadTokenService uploadTokenService, - FileMetadataService fileMetadataService, ContentServiceConfiguration contentServiceConfiguration) { - this.uploadTokenService = uploadTokenService; - this.fileMetadataService = fileMetadataService; - this.contentServiceConfiguration = contentServiceConfiguration; - this.storageProvider = storageProviderPool - .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); - } - - public UploadResponse writeDataChunk(UploadStore uploadInfo, byte[] dataChunk, int partId, boolean last) - throws Exception { - - if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // Upload hasn't started yet - uploadInfo = initializeUpload(uploadInfo); - } - - storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last); - - logger.info("Chunk {} was successfully written to output stream {}.", partId, uploadInfo.getFileName()); - uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); - if (!uploadTokenService.storeToken(uploadInfo, true)) { - // In this case data chunk was successfully appended to file (when LOCAL storage is used), but - // corresponding DB record contains incorrect data. This situation leads to data inconsistency. - // TODO: Check this behavior against cloud storage. (Google Cloud Storage). - logger.error( - "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); - return new UploadResponse( - "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", - UploadStatus.FAILED, uploadInfo.getPartId()); - - } - if (last) { - finalizeUpload(uploadInfo); - return new UploadResponse("File upload successfully completed.", UploadStatus.COMPLETED, - uploadInfo.getPartId()); - } - - return new UploadResponse("Data chunk upload successfully completed.", UploadStatus.IN_PROGRESS, - uploadInfo.getPartId()); - } - - private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { - String fileName = constructFileName(uploadInfo.getFileName()); - uploadInfo.setFileName(fileName); - - String uploadLocation = storageProvider.initialize(fileName); - uploadInfo.setUploadLocation(uploadLocation); - - return uploadInfo; - } - - public String finalizeUpload(UploadStore uploadInfo) { - - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, - storageProvider.getFileUrl(uploadInfo.getFileName()), storageProvider.getProviderType()); - String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); - uploadTokenService.deleteToken(uploadInfo.getUploadToken()); - logger.info("File sucessfully uploaded."); - return dowlnoadUrl; - } - - private String constructFileName(String fileName) { - String generatedPrefix = RandomStringUtils.randomAlphabetic(10); - return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); - } - - private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { - uploadInfo.setPartId(partId); - uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); - uploadInfo.setLastUploadedChunkTimestamp(Instant.now().toEpochMilli()); - uploadInfo.setStatus(UploadStatus.IN_PROGRESS); - return uploadInfo; - } - - private String constructDownloadUrl(String fileKey) { - return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); - } - - public boolean cancelUploadProcess(UploadStore uploadInfo) { - try { - this.storageProvider.close(uploadInfo.getUploadLocation()); - } catch (IOException e) { - logger.error("Error canceling upload job: {}", e.getMessage()); - logger.debug("Error canceling upload job: {}", e.getCause()); - return false; - } - uploadTokenService.deleteToken(uploadInfo.getUploadToken()); - return true; - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest; + +import java.io.IOException; +import java.time.Instant; + +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import biz.nynja.content.file.metadata.FileMetadataService; +import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.storage.StorageProviderPool; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class FileUploadService { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); + + private StorageProvider storageProvider; + private UploadTokenService uploadTokenService; + private FileMetadataService fileMetadataService; + private ContentServiceConfiguration contentServiceConfiguration; + + @Autowired + public FileUploadService(StorageProviderPool storageProviderPool, UploadTokenService uploadTokenService, + FileMetadataService fileMetadataService, ContentServiceConfiguration contentServiceConfiguration) { + this.uploadTokenService = uploadTokenService; + this.fileMetadataService = fileMetadataService; + this.contentServiceConfiguration = contentServiceConfiguration; + this.storageProvider = storageProviderPool + .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); + } + + public UploadResponse writeDataChunk(UploadStore uploadInfo, byte[] dataChunk, int partId, boolean last) + throws Exception { + + if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // Upload hasn't started yet + uploadInfo = initializeUpload(uploadInfo); + } + + storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last); + + logger.info("Chunk {} was successfully written to output stream {}.", partId, uploadInfo.getFileName()); + uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); + if (!uploadTokenService.storeToken(uploadInfo, true)) { + // In this case data chunk was successfully appended to file (when LOCAL storage is used), but + // corresponding DB record contains incorrect data. This situation leads to data inconsistency. + // TODO: Check this behavior against cloud storage. (Google Cloud Storage). + logger.error( + "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); + return new UploadResponse( + "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", + UploadStatus.FAILED, uploadInfo.getPartId()); + + } + if (last) { + finalizeUpload(uploadInfo); + return new UploadResponse("File upload successfully completed.", UploadStatus.COMPLETED, + uploadInfo.getPartId()); + } + + return new UploadResponse("Data chunk upload successfully completed.", UploadStatus.IN_PROGRESS, + uploadInfo.getPartId()); + } + + private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { + String fileName = constructFileName(uploadInfo.getFileName()); + uploadInfo.setFileName(fileName); + + String uploadLocation = storageProvider.initialize(fileName); + uploadInfo.setUploadLocation(uploadLocation); + + return uploadInfo; + } + + public String finalizeUpload(UploadStore uploadInfo) { + + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + storageProvider.getFileUrl(uploadInfo.getFileName()), storageProvider.getProviderType()); + String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + logger.info("File sucessfully uploaded."); + return dowlnoadUrl; + } + + private String constructFileName(String fileName) { + String generatedPrefix = RandomStringUtils.randomAlphabetic(10); + return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); + } + + private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { + uploadInfo.setPartId(partId); + uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); + uploadInfo.setLastUploadedChunkTimestamp(Instant.now().toEpochMilli()); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); + return uploadInfo; + } + + private String constructDownloadUrl(String fileKey) { + return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); + } + + public boolean cancelUploadProcess(UploadStore uploadInfo) { + try { + this.storageProvider.close(uploadInfo.getUploadLocation()); + } catch (IOException e) { + logger.error("Error canceling upload job: {}", e.getMessage()); + logger.debug("Error canceling upload job: {}", e.getCause()); + return false; + } + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + return true; + } +} diff --git a/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java b/src/main/java/biz/nynja/content/file/upload/rest/UploadResponse.java similarity index 93% rename from src/main/java/biz/nynja/content/upload/rest/UploadResponse.java rename to src/main/java/biz/nynja/content/file/upload/rest/UploadResponse.java index 0cb0766..0de3b12 100644 --- a/src/main/java/biz/nynja/content/upload/rest/UploadResponse.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/UploadResponse.java @@ -1,112 +1,112 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest; - -import biz.nynja.content.upload.models.UploadStatus; - -/** - * @author Ralitsa Todorova - * - */ -public class UploadResponse { - - private String message; - private UploadStatus status; - private int lastSuccessfulChunk; - private String downloadLink; - - public UploadResponse(String message) { - this.message = message; - } - - public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk) { - this.message = message; - this.status = status; - this.lastSuccessfulChunk = lastSuccessfulChunk; - } - - public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk, String downloadLink) { - this.message = message; - this.status = status; - this.lastSuccessfulChunk = lastSuccessfulChunk; - this.downloadLink = downloadLink; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public UploadStatus getStatus() { - return status; - } - - public void setStatus(UploadStatus status) { - this.status = status; - } - - public int getLastSuccessfulChunk() { - return lastSuccessfulChunk; - } - - public void setLastSuccessfulChunk(int lastSuccessfulChunk) { - this.lastSuccessfulChunk = lastSuccessfulChunk; - } - - public String getDownloadLink() { - return downloadLink; - } - - public void setDownloadLink(String downloadLink) { - this.downloadLink = downloadLink; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((downloadLink == null) ? 0 : downloadLink.hashCode()); - result = prime * result + lastSuccessfulChunk; - result = prime * result + ((message == null) ? 0 : message.hashCode()); - result = prime * result + ((status == null) ? 0 : status.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - UploadResponse other = (UploadResponse) obj; - if (downloadLink == null) { - if (other.downloadLink != null) - return false; - } else if (!downloadLink.equals(other.downloadLink)) - return false; - if (lastSuccessfulChunk != other.lastSuccessfulChunk) - return false; - if (message == null) { - if (other.message != null) - return false; - } else if (!message.equals(other.message)) - return false; - if (status != other.status) - return false; - return true; - } - - @Override - public String toString() { - return new StringBuilder("UploadResponse [message=").append(message).append(", status=").append(status) - .append(", lastSuccessfulChunk=").append(lastSuccessfulChunk).append(", downloadLink=") - .append(downloadLink).append("]").toString(); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest; + +import biz.nynja.content.file.upload.models.UploadStatus; + +/** + * @author Ralitsa Todorova + * + */ +public class UploadResponse { + + private String message; + private UploadStatus status; + private int lastSuccessfulChunk; + private String downloadLink; + + public UploadResponse(String message) { + this.message = message; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk, String downloadLink) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + this.downloadLink = downloadLink; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public UploadStatus getStatus() { + return status; + } + + public void setStatus(UploadStatus status) { + this.status = status; + } + + public int getLastSuccessfulChunk() { + return lastSuccessfulChunk; + } + + public void setLastSuccessfulChunk(int lastSuccessfulChunk) { + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public String getDownloadLink() { + return downloadLink; + } + + public void setDownloadLink(String downloadLink) { + this.downloadLink = downloadLink; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((downloadLink == null) ? 0 : downloadLink.hashCode()); + result = prime * result + lastSuccessfulChunk; + result = prime * result + ((message == null) ? 0 : message.hashCode()); + result = prime * result + ((status == null) ? 0 : status.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadResponse other = (UploadResponse) obj; + if (downloadLink == null) { + if (other.downloadLink != null) + return false; + } else if (!downloadLink.equals(other.downloadLink)) + return false; + if (lastSuccessfulChunk != other.lastSuccessfulChunk) + return false; + if (message == null) { + if (other.message != null) + return false; + } else if (!message.equals(other.message)) + return false; + if (status != other.status) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadResponse [message=").append(message).append(", status=").append(status) + .append(", lastSuccessfulChunk=").append(lastSuccessfulChunk).append(", downloadLink=") + .append(downloadLink).append("]").toString(); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java rename to src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java index a044084..22e0594 100644 --- a/src/main/java/biz/nynja/content/upload/rest/validation/ChunkUploadValidator.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java @@ -1,80 +1,80 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest.validation; - -import java.util.Optional; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import biz.nynja.content.upload.models.UploadStore; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class ChunkUploadValidator { - - private static final Logger logger = LoggerFactory.getLogger(ChunkUploadValidator.class); - - public Optional validateUploadRequest(UploadStore uploadInfo, byte[] data, String jobId, - String token, int partId, boolean last) { - - if (!jobId.equals(uploadInfo.getJobId().toString())) { - logger.error("Upload token {} does not apply to the requested job id: {}", token, jobId); - return Optional.of(new ValidationResult("Upload token does not apply to the requested upload URL.")); - } - - if (partId != uploadInfo.getPartId() + 1) { - logger.error( - "Chunk with Unexpected part id received for job id {} with associated token: {} . Expected: {}, actual: {}", - jobId, token, uploadInfo.getPartId() + 1, partId); - return Optional.of(new ValidationResult( - "You are trying to upload wrong data chunk. Last uploaded chunk " + uploadInfo.getPartId())); - } - if (data.length <= 0) { - logger.error("Recieved empty data chuck for jobId {} with associated token: {}", jobId, token); - return Optional.of(new ValidationResult("Missing chunk data.")); - } - - String sizeValidationResult = validateReachedFileSize(uploadInfo.getLastUploadedByte() + data.length, - uploadInfo.getFileSize(), last); - if (sizeValidationResult != null) { - logger.error("Incorrect file size reached for jobId {} with associated token: {}. {}", jobId, token, - sizeValidationResult); - return Optional.of(new ValidationResult(sizeValidationResult)); - } - - return Optional.empty(); - } - - private boolean matchesExpectedSize(int actual, int expected) { - return actual == expected; - } - - private boolean exceedsExpectedSize(int actual, int expected) { - return actual > expected; - } - - private String validateReachedFileSize(int actual, int expected, boolean lastChunk) { - if (exceedsExpectedSize(actual, expected)) { - logger.error("Size of uploaded file so far exceeds the expected file size."); - return "Size of uploaded file so far exceeds the expected file size."; - } - if (lastChunk && !matchesExpectedSize(actual, expected)) { - logger.error("Uploaded file size does not meet expectations - expected: {}, actual: {}.", expected, actual); - return "Uploaded file size does not meet expectations."; - } - if (!lastChunk && matchesExpectedSize(actual, expected)) { - logger.error( - "Uploaded file size so far reached expected overal file size, but this is not a final chunk - expected: {}, actual: {}.", - expected, actual); - return "Uploaded file size so far reached expected overal file size, but this is not a final chunk."; - } - return null; - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest.validation; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import biz.nynja.content.file.upload.models.UploadStore; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class ChunkUploadValidator { + + private static final Logger logger = LoggerFactory.getLogger(ChunkUploadValidator.class); + + public Optional validateUploadRequest(UploadStore uploadInfo, byte[] data, String jobId, + String token, int partId, boolean last) { + + if (!jobId.equals(uploadInfo.getJobId().toString())) { + logger.error("Upload token {} does not apply to the requested job id: {}", token, jobId); + return Optional.of(new ValidationResult("Upload token does not apply to the requested upload URL.")); + } + + if (partId != uploadInfo.getPartId() + 1) { + logger.error( + "Chunk with Unexpected part id received for job id {} with associated token: {} . Expected: {}, actual: {}", + jobId, token, uploadInfo.getPartId() + 1, partId); + return Optional.of(new ValidationResult( + "You are trying to upload wrong data chunk. Last uploaded chunk " + uploadInfo.getPartId())); + } + if (data.length <= 0) { + logger.error("Recieved empty data chuck for jobId {} with associated token: {}", jobId, token); + return Optional.of(new ValidationResult("Missing chunk data.")); + } + + String sizeValidationResult = validateReachedFileSize(uploadInfo.getLastUploadedByte() + data.length, + uploadInfo.getFileSize(), last); + if (sizeValidationResult != null) { + logger.error("Incorrect file size reached for jobId {} with associated token: {}. {}", jobId, token, + sizeValidationResult); + return Optional.of(new ValidationResult(sizeValidationResult)); + } + + return Optional.empty(); + } + + private boolean matchesExpectedSize(int actual, int expected) { + return actual == expected; + } + + private boolean exceedsExpectedSize(int actual, int expected) { + return actual > expected; + } + + private String validateReachedFileSize(int actual, int expected, boolean lastChunk) { + if (exceedsExpectedSize(actual, expected)) { + logger.error("Size of uploaded file so far exceeds the expected file size."); + return "Size of uploaded file so far exceeds the expected file size."; + } + if (lastChunk && !matchesExpectedSize(actual, expected)) { + logger.error("Uploaded file size does not meet expectations - expected: {}, actual: {}.", expected, actual); + return "Uploaded file size does not meet expectations."; + } + if (!lastChunk && matchesExpectedSize(actual, expected)) { + logger.error( + "Uploaded file size so far reached expected overal file size, but this is not a final chunk - expected: {}, actual: {}.", + expected, actual); + return "Uploaded file size so far reached expected overal file size, but this is not a final chunk."; + } + return null; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java b/src/main/java/biz/nynja/content/file/upload/rest/validation/ValidationResult.java similarity index 85% rename from src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java rename to src/main/java/biz/nynja/content/file/upload/rest/validation/ValidationResult.java index 5074bd9..288e0a4 100644 --- a/src/main/java/biz/nynja/content/upload/rest/validation/ValidationResult.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/validation/ValidationResult.java @@ -1,26 +1,26 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.rest.validation; - -/** - * @author Ralitsa Todorova - * - */ -public class ValidationResult { - - private String message; - - public ValidationResult(String message) { - this.message = message; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest.validation; + +/** + * @author Ralitsa Todorova + * + */ +public class ValidationResult { + + private String message; + + public ValidationResult(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadToken.java b/src/main/java/biz/nynja/content/file/upload/token/UploadToken.java similarity index 95% rename from src/main/java/biz/nynja/content/upload/token/UploadToken.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadToken.java index 90ad1c0..ae63841 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadToken.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadToken.java @@ -1,124 +1,124 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -/** - * @author Ralitsa Todorova - * - */ -public class UploadToken { - - private String name; - private String mediaType; - private int size; - private String uploadedFrom; - private String deviceId; - - public UploadToken(String name, String mediaType, int size, String uploadedFrom, String deviceId) { - this.name = name; - this.mediaType = mediaType; - this.size = size; - this.uploadedFrom = uploadedFrom; - this.deviceId = deviceId; - } - - public String getDeviceId() { - return deviceId; - } - - public void setDeviceId(String deviceId) { - this.deviceId = deviceId; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getMediaType() { - return mediaType; - } - - public void setMediaType(String mediaType) { - this.mediaType = mediaType; - } - - public int getSize() { - return size; - } - - public void setSize(int size) { - this.size = size; - } - - public String getUploadedFrom() { - return uploadedFrom; - } - - public void setUploadedFrom(String uploadedFrom) { - this.uploadedFrom = uploadedFrom; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); - result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); - result = prime * result + ((name == null) ? 0 : name.hashCode()); - result = prime * result + size; - result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - UploadToken other = (UploadToken) obj; - if (deviceId == null) { - if (other.deviceId != null) - return false; - } else if (!deviceId.equals(other.deviceId)) - return false; - if (mediaType == null) { - if (other.mediaType != null) - return false; - } else if (!mediaType.equals(other.mediaType)) - return false; - if (name == null) { - if (other.name != null) - return false; - } else if (!name.equals(other.name)) - return false; - if (size != other.size) - return false; - if (uploadedFrom == null) { - if (other.uploadedFrom != null) - return false; - } else if (!uploadedFrom.equals(other.uploadedFrom)) - return false; - return true; - } - - @Override - public String toString() { - return new StringBuilder("UploadToken [name=").append(name).append(", mediaType=").append(mediaType) - .append(", size=").append(size).append(", uploadedFrom=").append(uploadedFrom).append(", deviceId=") - .append(deviceId).append("]").toString(); - } - - public String getTokenString() { - return new StringBuilder("n:").append(this.getName()).append(",mt:").append(this.getMediaType()).append(",s:") - .append(this.getSize()).append(",uf:").append(this.getUploadedFrom()).append(",did:") - .append(this.getDeviceId()).toString(); - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +/** + * @author Ralitsa Todorova + * + */ +public class UploadToken { + + private String name; + private String mediaType; + private int size; + private String uploadedFrom; + private String deviceId; + + public UploadToken(String name, String mediaType, int size, String uploadedFrom, String deviceId) { + this.name = name; + this.mediaType = mediaType; + this.size = size; + this.uploadedFrom = uploadedFrom; + this.deviceId = deviceId; + } + + public String getDeviceId() { + return deviceId; + } + + public void setDeviceId(String deviceId) { + this.deviceId = deviceId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getMediaType() { + return mediaType; + } + + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public String getUploadedFrom() { + return uploadedFrom; + } + + public void setUploadedFrom(String uploadedFrom) { + this.uploadedFrom = uploadedFrom; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); + result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + size; + result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadToken other = (UploadToken) obj; + if (deviceId == null) { + if (other.deviceId != null) + return false; + } else if (!deviceId.equals(other.deviceId)) + return false; + if (mediaType == null) { + if (other.mediaType != null) + return false; + } else if (!mediaType.equals(other.mediaType)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (size != other.size) + return false; + if (uploadedFrom == null) { + if (other.uploadedFrom != null) + return false; + } else if (!uploadedFrom.equals(other.uploadedFrom)) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadToken [name=").append(name).append(", mediaType=").append(mediaType) + .append(", size=").append(size).append(", uploadedFrom=").append(uploadedFrom).append(", deviceId=") + .append(deviceId).append("]").toString(); + } + + public String getTokenString() { + return new StringBuilder("n:").append(this.getName()).append(",mt:").append(this.getMediaType()).append(",s:") + .append(this.getSize()).append(",uf:").append(this.getUploadedFrom()).append(",did:") + .append(this.getDeviceId()).toString(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadTokenResponseProvider.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenResponseProvider.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/token/UploadTokenResponseProvider.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadTokenResponseProvider.java index 7cfb9b9..1582468 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadTokenResponseProvider.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenResponseProvider.java @@ -1,34 +1,34 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -import org.springframework.stereotype.Service; - -import biz.nynja.content.grpc.ErrorResponse; -import biz.nynja.content.grpc.ErrorResponse.Cause; -import biz.nynja.content.grpc.UploadToken; -import biz.nynja.content.grpc.UploadTokenResponse; -import io.grpc.stub.StreamObserver; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class UploadTokenResponseProvider { - - public void prepareUploadTokenResponse(StreamObserver responseObserver, String token, - String tokenUrl) { - responseObserver.onNext(UploadTokenResponse.newBuilder() - .setTokenDetails(UploadToken.newBuilder().setToken(token).setUploadUrl(tokenUrl)).build()); - responseObserver.onCompleted(); - } - - public void prepareErrorResponse(StreamObserver responseObserver, Cause error, - String message) { - responseObserver.onNext(UploadTokenResponse.newBuilder() - .setError(ErrorResponse.newBuilder().setCause(error).setMessage(message)).build()); - responseObserver.onCompleted(); - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +import org.springframework.stereotype.Service; + +import biz.nynja.content.grpc.ErrorResponse; +import biz.nynja.content.grpc.ErrorResponse.Cause; +import biz.nynja.content.grpc.UploadToken; +import biz.nynja.content.grpc.UploadTokenResponse; +import io.grpc.stub.StreamObserver; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class UploadTokenResponseProvider { + + public void prepareUploadTokenResponse(StreamObserver responseObserver, String token, + String tokenUrl) { + responseObserver.onNext(UploadTokenResponse.newBuilder() + .setTokenDetails(UploadToken.newBuilder().setToken(token).setUploadUrl(tokenUrl)).build()); + responseObserver.onCompleted(); + } + + public void prepareErrorResponse(StreamObserver responseObserver, Cause error, + String message) { + responseObserver.onNext(UploadTokenResponse.newBuilder() + .setError(ErrorResponse.newBuilder().setCause(error).setMessage(message)).build()); + responseObserver.onCompleted(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java similarity index 91% rename from src/main/java/biz/nynja/content/upload/token/UploadTokenService.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index f11abfd..fe306de 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -1,101 +1,101 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -import java.io.UnsupportedEncodingException; -import java.security.InvalidAlgorithmParameterException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.util.Optional; - -import javax.crypto.BadPaddingException; -import javax.crypto.IllegalBlockSizeException; -import javax.crypto.NoSuchPaddingException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import biz.nynja.content.security.EncryptDecryptService; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.repositories.UploadStoreRepository; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class UploadTokenService { - - private final Logger logger = LoggerFactory.getLogger(UploadTokenService.class); - - private EncryptDecryptService encryptDecryptService; - private UploadStoreRepository uploadStoreRepository; - private UploadTokenConfiguration uploadTokenConfiguration; - - public UploadTokenService(EncryptDecryptService encryptDecryptService, - UploadStoreRepository uploadStoreRepository, UploadTokenConfiguration uploadTokenConfiguration) { - this.encryptDecryptService = encryptDecryptService; - this.uploadStoreRepository = uploadStoreRepository; - this.uploadTokenConfiguration = uploadTokenConfiguration; - } - - public Optional prepareTokenResponse(UploadToken uploadToken) { - String result; - try { - result = encryptDecryptService.encrypt(uploadToken.getTokenString()); - logger.debug("Upload token successfully encrypted: {}", uploadToken.toString()); - } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException - | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException - | UnsupportedEncodingException e) { - logger.error("Error encrypting upload token. {}", e.getCause()); - logger.debug("Error encrypting upload token: {}", e.getMessage()); - return Optional.empty(); - } - - return Optional.of(result); - } - - public boolean storeToken(UploadStore uploadInfo, boolean permanent) { - try { - if (permanent) { - uploadStoreRepository.save(uploadInfo); - logger.info("Upload token successfully stored in DB."); - } else { - uploadStoreRepository.saveWithTtl(uploadInfo, uploadTokenConfiguration.getTokenTimeToLive()); - logger.info("Upload token successfully stored in DB for {} seconds.", - uploadTokenConfiguration.getTokenTimeToLive()); - } - logger.info("Upload token successfully stored in DB."); - return true; - } catch (Exception e) { - logger.error("Error storing record {} in DB: {}", uploadInfo.toString(), e.getMessage()); - logger.debug("Error storing record {} in DB: {}", uploadInfo.toString(), e.getCause()); - return false; - } - } - - public void deleteToken(String tokenId) { - try { - logger.debug("Deleting token {} from DB...", tokenId); - uploadStoreRepository.deleteById(tokenId); - logger.debug("Token {} successfully deleted from DB.", tokenId); - } catch (Exception e) { - logger.error("Error deleting token {} from DB: {}", tokenId, e.getMessage()); - logger.debug("Error deleting token {} from DB: {}", tokenId, e.getCause()); - } - - uploadStoreRepository.deleteById(tokenId); - } - - public Optional getUploadInfo(String uploadToken) { - - UploadStore storeInfo = uploadStoreRepository.findByUploadToken(uploadToken); - if (storeInfo != null) { - return Optional.of(storeInfo); - } - return Optional.empty(); - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +import java.io.UnsupportedEncodingException; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.Optional; + +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import biz.nynja.content.security.EncryptDecryptService; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.repositories.UploadStoreRepository; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class UploadTokenService { + + private final Logger logger = LoggerFactory.getLogger(UploadTokenService.class); + + private EncryptDecryptService encryptDecryptService; + private UploadStoreRepository uploadStoreRepository; + private UploadTokenConfiguration uploadTokenConfiguration; + + public UploadTokenService(EncryptDecryptService encryptDecryptService, + UploadStoreRepository uploadStoreRepository, UploadTokenConfiguration uploadTokenConfiguration) { + this.encryptDecryptService = encryptDecryptService; + this.uploadStoreRepository = uploadStoreRepository; + this.uploadTokenConfiguration = uploadTokenConfiguration; + } + + public Optional prepareTokenResponse(UploadToken uploadToken) { + String result; + try { + result = encryptDecryptService.encrypt(uploadToken.getTokenString()); + logger.debug("Upload token successfully encrypted: {}", uploadToken.toString()); + } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException + | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException + | UnsupportedEncodingException e) { + logger.error("Error encrypting upload token. {}", e.getCause()); + logger.debug("Error encrypting upload token: {}", e.getMessage()); + return Optional.empty(); + } + + return Optional.of(result); + } + + public boolean storeToken(UploadStore uploadInfo, boolean permanent) { + try { + if (permanent) { + uploadStoreRepository.save(uploadInfo); + logger.info("Upload token successfully stored in DB."); + } else { + uploadStoreRepository.saveWithTtl(uploadInfo, uploadTokenConfiguration.getTokenTimeToLive()); + logger.info("Upload token successfully stored in DB for {} seconds.", + uploadTokenConfiguration.getTokenTimeToLive()); + } + logger.info("Upload token successfully stored in DB."); + return true; + } catch (Exception e) { + logger.error("Error storing record {} in DB: {}", uploadInfo.toString(), e.getMessage()); + logger.debug("Error storing record {} in DB: {}", uploadInfo.toString(), e.getCause()); + return false; + } + } + + public void deleteToken(String tokenId) { + try { + logger.debug("Deleting token {} from DB...", tokenId); + uploadStoreRepository.deleteById(tokenId); + logger.debug("Token {} successfully deleted from DB.", tokenId); + } catch (Exception e) { + logger.error("Error deleting token {} from DB: {}", tokenId, e.getMessage()); + logger.debug("Error deleting token {} from DB: {}", tokenId, e.getCause()); + } + + uploadStoreRepository.deleteById(tokenId); + } + + public Optional getUploadInfo(String uploadToken) { + + UploadStore storeInfo = uploadStoreRepository.findByUploadToken(uploadToken); + if (storeInfo != null) { + return Optional.of(storeInfo); + } + return Optional.empty(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadTokenValidator.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenValidator.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/token/UploadTokenValidator.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadTokenValidator.java index a5a1f24..934838e 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadTokenValidator.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenValidator.java @@ -1,127 +1,127 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -import java.util.Collection; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; - -import com.google.common.io.Files; - -import biz.nynja.content.grpc.ErrorResponse.Cause; -import biz.nynja.content.core.validation.Validation; -import biz.nynja.content.core.validation.ValidationError; -import biz.nynja.content.grpc.UploadTokenRequest; -import biz.nynja.content.grpc.UploadTokenRequest.MediaType; -import biz.nynja.content.upload.token.configuration.MediaTypesConfiguration; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class UploadTokenValidator { - - private MediaTypesConfiguration mediaTypesConfiguration; - private UploadTokenConfiguration uploadTokenConfiguration; - - @Autowired - public UploadTokenValidator(MediaTypesConfiguration mediaTypesConfiguration, - UploadTokenConfiguration uploadTokenConfiguration) { - this.mediaTypesConfiguration = mediaTypesConfiguration; - this.uploadTokenConfiguration = uploadTokenConfiguration; - } - - public Validation validateGetTokenRequest(UploadTokenRequest request) { - Validation validation = new Validation(); - if (StringUtils.isEmpty(request.getName())) { - validation.addError(new ValidationError("Missing file name", Cause.MISSING_NAME)); - } - if (request.getSize() <= 0 || request.getSize() > uploadTokenConfiguration.getFileMaxSize()) { - validation.addError(new ValidationError("Incorrect file size", Cause.INCORRECT_SIZE)); - } - if (request.getMediaType().equals(MediaType.UNKNOWN_MEDIA_TYPE)) { - validation.addError(new ValidationError("Missing media type", Cause.MISSING_MEDIA_TYPE)); - } else { - String fileExtension = Files.getFileExtension(request.getName()); - Validation mediaTypeValidation = validateMediaType(request.getMediaType(), fileExtension); - if (mediaTypeValidation.hasErrors()) { - validation.addErrors(mediaTypeValidation.getErrors()); - } - } - if (StringUtils.isEmpty(request.getUploadedFrom())) { - validation.addError(new ValidationError("Missing uploadedFrom information", Cause.MISSING_UPLOADED_FROM)); - } - if (StringUtils.isEmpty(request.getDeviceId())) { - validation.addError(new ValidationError("Missing device id.", Cause.MISSING_DEVICE_ID)); - } - return validation; - } - - private Validation validateMediaType(MediaType requestMediaType, String fileExtension) { - Validation validation = new Validation(); - switch (requestMediaType) { - case AUDIO: - if (!mediaTypesConfiguration.getAudio().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case VIDEO: - if (!mediaTypesConfiguration.getVideo().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case COMPRESSED_FILES: - if (!mediaTypesConfiguration.getCompressed().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case DATA_FILES: - if (!mediaTypesConfiguration.getData().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case IMAGE: - if (!mediaTypesConfiguration.getImage().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case PAGE_LAYOUT: - if (!mediaTypesConfiguration.getPageLayout().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case SPREADSHEET: - if (!mediaTypesConfiguration.getSpreadsheet().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case TEXT: - if (!mediaTypesConfiguration.getText().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case OTHER: - if (Stream - .of(mediaTypesConfiguration.getAudio(), mediaTypesConfiguration.getData(), - mediaTypesConfiguration.getCompressed(), mediaTypesConfiguration.getImage(), - mediaTypesConfiguration.getPageLayout(), mediaTypesConfiguration.getSpreadsheet(), - mediaTypesConfiguration.getText(), mediaTypesConfiguration.getVideo()) - .flatMap(Collection::stream).collect(Collectors.toList()).contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - default: - break; - } - - return validation; - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import com.google.common.io.Files; + +import biz.nynja.content.grpc.ErrorResponse.Cause; +import biz.nynja.content.core.validation.Validation; +import biz.nynja.content.core.validation.ValidationError; +import biz.nynja.content.grpc.UploadTokenRequest; +import biz.nynja.content.grpc.UploadTokenRequest.MediaType; +import biz.nynja.content.file.upload.token.configuration.MediaTypesConfiguration; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class UploadTokenValidator { + + private MediaTypesConfiguration mediaTypesConfiguration; + private UploadTokenConfiguration uploadTokenConfiguration; + + @Autowired + public UploadTokenValidator(MediaTypesConfiguration mediaTypesConfiguration, + UploadTokenConfiguration uploadTokenConfiguration) { + this.mediaTypesConfiguration = mediaTypesConfiguration; + this.uploadTokenConfiguration = uploadTokenConfiguration; + } + + public Validation validateGetTokenRequest(UploadTokenRequest request) { + Validation validation = new Validation(); + if (StringUtils.isEmpty(request.getName())) { + validation.addError(new ValidationError("Missing file name", Cause.MISSING_NAME)); + } + if (request.getSize() <= 0 || request.getSize() > uploadTokenConfiguration.getFileMaxSize()) { + validation.addError(new ValidationError("Incorrect file size", Cause.INCORRECT_SIZE)); + } + if (request.getMediaType().equals(MediaType.UNKNOWN_MEDIA_TYPE)) { + validation.addError(new ValidationError("Missing media type", Cause.MISSING_MEDIA_TYPE)); + } else { + String fileExtension = Files.getFileExtension(request.getName()); + Validation mediaTypeValidation = validateMediaType(request.getMediaType(), fileExtension); + if (mediaTypeValidation.hasErrors()) { + validation.addErrors(mediaTypeValidation.getErrors()); + } + } + if (StringUtils.isEmpty(request.getUploadedFrom())) { + validation.addError(new ValidationError("Missing uploadedFrom information", Cause.MISSING_UPLOADED_FROM)); + } + if (StringUtils.isEmpty(request.getDeviceId())) { + validation.addError(new ValidationError("Missing device id.", Cause.MISSING_DEVICE_ID)); + } + return validation; + } + + private Validation validateMediaType(MediaType requestMediaType, String fileExtension) { + Validation validation = new Validation(); + switch (requestMediaType) { + case AUDIO: + if (!mediaTypesConfiguration.getAudio().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case VIDEO: + if (!mediaTypesConfiguration.getVideo().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case COMPRESSED_FILES: + if (!mediaTypesConfiguration.getCompressed().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case DATA_FILES: + if (!mediaTypesConfiguration.getData().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case IMAGE: + if (!mediaTypesConfiguration.getImage().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case PAGE_LAYOUT: + if (!mediaTypesConfiguration.getPageLayout().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case SPREADSHEET: + if (!mediaTypesConfiguration.getSpreadsheet().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case TEXT: + if (!mediaTypesConfiguration.getText().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case OTHER: + if (Stream + .of(mediaTypesConfiguration.getAudio(), mediaTypesConfiguration.getData(), + mediaTypesConfiguration.getCompressed(), mediaTypesConfiguration.getImage(), + mediaTypesConfiguration.getPageLayout(), mediaTypesConfiguration.getSpreadsheet(), + mediaTypesConfiguration.getText(), mediaTypesConfiguration.getVideo()) + .flatMap(Collection::stream).collect(Collectors.toList()).contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + default: + break; + } + + return validation; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/token/configuration/MediaTypesConfiguration.java b/src/main/java/biz/nynja/content/file/upload/token/configuration/MediaTypesConfiguration.java similarity index 96% rename from src/main/java/biz/nynja/content/upload/token/configuration/MediaTypesConfiguration.java rename to src/main/java/biz/nynja/content/file/upload/token/configuration/MediaTypesConfiguration.java index 9f0b431..e95a404 100644 --- a/src/main/java/biz/nynja/content/upload/token/configuration/MediaTypesConfiguration.java +++ b/src/main/java/biz/nynja/content/file/upload/token/configuration/MediaTypesConfiguration.java @@ -1,7 +1,7 @@ /** * Copyright (C) 2018 Nynja Inc. All rights reserved. */ -package biz.nynja.content.upload.token.configuration; +package biz.nynja.content.file.upload.token.configuration; import java.util.List; diff --git a/src/main/java/biz/nynja/content/upload/token/configuration/UploadTokenConfiguration.java b/src/main/java/biz/nynja/content/file/upload/token/configuration/UploadTokenConfiguration.java similarity index 92% rename from src/main/java/biz/nynja/content/upload/token/configuration/UploadTokenConfiguration.java rename to src/main/java/biz/nynja/content/file/upload/token/configuration/UploadTokenConfiguration.java index 27c1f81..54aabfc 100644 --- a/src/main/java/biz/nynja/content/upload/token/configuration/UploadTokenConfiguration.java +++ b/src/main/java/biz/nynja/content/file/upload/token/configuration/UploadTokenConfiguration.java @@ -1,51 +1,51 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token.configuration; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; - -/** - * @author Ralitsa Todorova - * - */ - -@Configuration -public class UploadTokenConfiguration { - - private final int tokenTimeToLive; - private final int fileMaxSize; - private final int maxRetries; - - @Autowired - public UploadTokenConfiguration(Environment env) { - this.tokenTimeToLive = parseProperty(env, "token.time_to_live"); - this.fileMaxSize = parseProperty(env, "token.max_file_size"); - this.maxRetries = parseProperty(env, "token.max_upload_retries"); - } - - private int parseProperty(Environment env, String property) throws InternalError { - int intProperty = 0; - try { - intProperty = Integer.parseInt(env.getRequiredProperty(property)); - } catch (NumberFormatException ex) { - throw new InternalError("Integer expected in " + property); - } - return intProperty; - - } - - public int getTokenTimeToLive() { - return tokenTimeToLive; - } - - public int getFileMaxSize() { - return fileMaxSize; - } - - public int getMaxRetries() { - return maxRetries; - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token.configuration; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +/** + * @author Ralitsa Todorova + * + */ + +@Configuration +public class UploadTokenConfiguration { + + private final int tokenTimeToLive; + private final int fileMaxSize; + private final int maxRetries; + + @Autowired + public UploadTokenConfiguration(Environment env) { + this.tokenTimeToLive = parseProperty(env, "token.time_to_live"); + this.fileMaxSize = parseProperty(env, "token.max_file_size"); + this.maxRetries = parseProperty(env, "token.max_upload_retries"); + } + + private int parseProperty(Environment env, String property) throws InternalError { + int intProperty = 0; + try { + intProperty = Integer.parseInt(env.getRequiredProperty(property)); + } catch (NumberFormatException ex) { + throw new InternalError("Integer expected in " + property); + } + return intProperty; + + } + + public int getTokenTimeToLive() { + return tokenTimeToLive; + } + + public int getFileMaxSize() { + return fileMaxSize; + } + + public int getMaxRetries() { + return maxRetries; + } +} diff --git a/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java b/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java index 9cfeda2..7c91613 100644 --- a/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java +++ b/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java @@ -11,9 +11,9 @@ import org.slf4j.LoggerFactory; import org.springframework.util.StringUtils; import biz.nynja.content.grpc.constants.ContentServiceConstants; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadTokenService; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; import io.grpc.Context; import io.grpc.Contexts; import io.grpc.Metadata; diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index efc6978..5713b2a 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -32,13 +32,13 @@ import biz.nynja.content.grpc.UploadTokenResponse; import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; import biz.nynja.content.grpc.constants.ContentServiceConstants; import biz.nynja.content.grpc.interceptors.UploadInterceptor; -import biz.nynja.content.upload.models.UploadStatus; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadToken; -import biz.nynja.content.upload.token.UploadTokenResponseProvider; -import biz.nynja.content.upload.token.UploadTokenService; -import biz.nynja.content.upload.token.UploadTokenValidator; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadToken; +import biz.nynja.content.file.upload.token.UploadTokenResponseProvider; +import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.token.UploadTokenValidator; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; import io.grpc.Status; import io.grpc.StatusException; import io.grpc.stub.StreamObserver; diff --git a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java index 3683322..ad3ce0c 100644 --- a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java +++ b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java @@ -27,13 +27,15 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; +import biz.nynja.content.file.FileControllersExceptionHandler; + /** * @author Angel.Botev * */ @RunWith(SpringRunner.class) @WebMvcTest(FileDownloadController.class) -@ContextConfiguration(classes = { FileDownloadController.class, FileControllerExceptionHandler.class }) +@ContextConfiguration(classes = { FileDownloadController.class, FileControllersExceptionHandler.class }) @ActiveProfiles("dev") public class FileDownloadControllerTest { diff --git a/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java index 0963bde..f28902d 100644 --- a/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java +++ b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java @@ -24,13 +24,13 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; -import biz.nynja.content.upload.models.UploadStatus; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.rest.FileUploadController; -import biz.nynja.content.upload.rest.FileUploadService; -import biz.nynja.content.upload.rest.UploadResponse; -import biz.nynja.content.upload.rest.validation.ChunkUploadValidator; -import biz.nynja.content.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.FileUploadController; +import biz.nynja.content.file.upload.rest.FileUploadService; +import biz.nynja.content.file.upload.rest.UploadResponse; +import biz.nynja.content.file.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.file.upload.token.UploadTokenService; /** * @author Ralitsa Todorova diff --git a/src/test/java/biz/nynja/content/file/upload/Util.java b/src/test/java/biz/nynja/content/file/upload/Util.java index 9e9085a..47f42fb 100644 --- a/src/test/java/biz/nynja/content/file/upload/Util.java +++ b/src/test/java/biz/nynja/content/file/upload/Util.java @@ -9,9 +9,9 @@ import java.util.UUID; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; -import biz.nynja.content.upload.models.UploadStatus; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.rest.UploadResponse; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.UploadResponse; /** * @author Ralitsa Todorova diff --git a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java index 631d9ab..5eee2dc 100644 --- a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java +++ b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java @@ -26,9 +26,9 @@ import biz.nynja.content.grpc.ErrorResponse; import biz.nynja.content.grpc.UploadTokenRequest; import biz.nynja.content.grpc.UploadTokenResponse; import biz.nynja.content.grpc.services.util.ContentServiceUtil; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadToken; -import biz.nynja.content.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadToken; +import biz.nynja.content.file.upload.token.UploadTokenService; import io.grpc.Metadata; import io.grpc.stub.MetadataUtils; diff --git a/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java b/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java index 07ec4cc..1a6f12d 100644 --- a/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java +++ b/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java @@ -27,8 +27,10 @@ import org.springframework.test.context.junit4.SpringRunner; import biz.nynja.content.Util; import biz.nynja.content.security.EncryptDecryptConfiguration; import biz.nynja.content.security.EncryptDecryptService; -import biz.nynja.content.upload.repositories.UploadStoreRepository; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.file.upload.repositories.UploadStoreRepository; +import biz.nynja.content.file.upload.token.UploadToken; +import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; /** * @author Ralitsa Todorova -- GitLab From 9aacb28dc016a0f62b8f48e4319d9b71ac2e8347 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 29 Jan 2019 18:10:18 +0200 Subject: [PATCH 24/29] NY-6837 Remove old upload jobs - add restcontroller; - add delete old jobs function; - add getOldJobs function; Signed-off-by: abotev-intracol --- .../biz/nynja/content/file/JobScheduler.java | 42 ++++++++++++++++++ .../file/upload/token/UploadTokenService.java | 44 +++++++++++++++++-- .../ContentServiceConfiguration.java | 17 +++++++ src/main/resources/application-dev.yml | 3 ++ src/main/resources/application-production.yml | 2 + 5 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 src/main/java/biz/nynja/content/file/JobScheduler.java diff --git a/src/main/java/biz/nynja/content/file/JobScheduler.java b/src/main/java/biz/nynja/content/file/JobScheduler.java new file mode 100644 index 0000000..f2e8fea --- /dev/null +++ b/src/main/java/biz/nynja/content/file/JobScheduler.java @@ -0,0 +1,42 @@ +package biz.nynja.content.file; + +import java.util.List; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.RestController; + +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.FileUploadController; +import biz.nynja.content.file.upload.token.UploadTokenService; + +@RestController +public class JobScheduler { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); + + private UploadTokenService uploadTokenService; + + public JobScheduler(UploadTokenService uploadTokenService) { + this.uploadTokenService = uploadTokenService; + } + + @DeleteMapping("/jobs/old") + public ResponseEntity removeOldJobs() { + logger.info("Remove not finished (old) jobs "); + + Optional> oldJobs = uploadTokenService.getOldUploadJobs(); + if (oldJobs.isPresent()) { + logger.info("Founded old jobs! Countinue with delete process", oldJobs); + uploadTokenService.deleteJobs(oldJobs.get()); + return new ResponseEntity("Old jobs successfully deleted!", HttpStatus.NO_CONTENT); + } else { + logger.info("Old jobs didn't found!"); + return new ResponseEntity("Old jobs didn't found!", HttpStatus.NO_CONTENT); + } + } +} diff --git a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index fe306de..c651136 100644 --- a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -7,6 +7,10 @@ import java.io.UnsupportedEncodingException; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import javax.crypto.BadPaddingException; @@ -17,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; import biz.nynja.content.security.EncryptDecryptService; import biz.nynja.content.file.upload.models.UploadStore; import biz.nynja.content.file.upload.repositories.UploadStoreRepository; @@ -34,12 +39,15 @@ public class UploadTokenService { private EncryptDecryptService encryptDecryptService; private UploadStoreRepository uploadStoreRepository; private UploadTokenConfiguration uploadTokenConfiguration; + private ContentServiceConfiguration contentServiceConfiguration; - public UploadTokenService(EncryptDecryptService encryptDecryptService, - UploadStoreRepository uploadStoreRepository, UploadTokenConfiguration uploadTokenConfiguration) { + public UploadTokenService(EncryptDecryptService encryptDecryptService, UploadStoreRepository uploadStoreRepository, + UploadTokenConfiguration uploadTokenConfiguration, + ContentServiceConfiguration contentServiceConfiguration) { this.encryptDecryptService = encryptDecryptService; this.uploadStoreRepository = uploadStoreRepository; this.uploadTokenConfiguration = uploadTokenConfiguration; + this.contentServiceConfiguration = contentServiceConfiguration; } public Optional prepareTokenResponse(UploadToken uploadToken) { @@ -86,10 +94,20 @@ public class UploadTokenService { logger.error("Error deleting token {} from DB: {}", tokenId, e.getMessage()); logger.debug("Error deleting token {} from DB: {}", tokenId, e.getCause()); } + } - uploadStoreRepository.deleteById(tokenId); + public void deleteJobs(List jobs) { + try { + logger.debug("Deleting jobs {} from DB...", jobs); + uploadStoreRepository.deleteAll(jobs); + logger.debug("Jobs {} successfully deleted from DB.", jobs); + } catch (Exception e) { + logger.error("Error deleting jobs {} from DB: {}", jobs, e.getMessage()); + logger.debug("Error deleting jobs {} from DB: {}", jobs, e.getCause()); + } } + public Optional getUploadInfo(String uploadToken) { UploadStore storeInfo = uploadStoreRepository.findByUploadToken(uploadToken); @@ -98,4 +116,24 @@ public class UploadTokenService { } return Optional.empty(); } + + public Optional> getOldUploadJobs() { + + List jobs = uploadStoreRepository.findAll(); + if (jobs == null) { + return Optional.empty(); + } + + List oldJobs = new ArrayList<>(); + int jobTTL = contentServiceConfiguration.getJobTTL(); + Instant now = Instant.now(); + for (UploadStore job : jobs) { + Instant last = Instant.ofEpochMilli(job.getLastUploadedChunkTimestamp()); + long hours = ChronoUnit.HOURS.between(now, last); + if (Math.toIntExact(hours) > jobTTL) { + oldJobs.add(job); + } + } + return Optional.of(oldJobs); + } } diff --git a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java index e520288..e37e53f 100644 --- a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java +++ b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java @@ -19,12 +19,14 @@ public class ContentServiceConfiguration { private final String uploadUrl; private final String downloadUrl; private final Provider storageProvider; + private final int jobTTL; @Autowired public ContentServiceConfiguration(Environment env) { this.uploadUrl = env.getRequiredProperty("file.upload.url"); this.downloadUrl = env.getRequiredProperty("file.download.url"); this.storageProvider = getProvidersProperty("storage.provider", env); + this.jobTTL = parseProperty(env, "file.upload.job.ttl"); } private Provider getProvidersProperty(String key, Environment env) { @@ -32,6 +34,17 @@ public class ContentServiceConfiguration { return Provider.valueOf(provider); } + private int parseProperty(Environment env, String property) throws InternalError { + int intProperty = 0; + try { + intProperty = Integer.parseInt(env.getRequiredProperty(property)); + } catch (NumberFormatException ex) { + throw new InternalError("Integer expected in " + property); + } + return intProperty; + + } + public String getUploadUrl() { return uploadUrl; } @@ -43,4 +56,8 @@ public class ContentServiceConfiguration { public Provider getStorageProvider() { return storageProvider; } + + public int getJobTTL() { + return jobTTL; + } } diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 77123f4..b1f98b2 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -37,6 +37,8 @@ media-types: file: upload: url: http://localhost:${server.port}/file/upload/ + job: + ttl: 24 # measured in hours. download: url: http://localhost:${server.port}/file/download/ @@ -49,6 +51,7 @@ storage: bucket: content-service-dev upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + # To enable colors in Eclipse: # spring.output.ansi.enabled=ALWAYS and in eclipse # Help -> Install New Software... and click "Add..." to add the following URL: diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 29d8d6c..47bd9a5 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -37,6 +37,8 @@ media-types: file: upload: url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net/file/upload} + job: + ttl: 24 # measured in hours. download: url: ${FILE_DOWNLOAD_URL:https://content.dev-eu.nynja.net/file/download/} -- GitLab From 2b69bc3416762de6c725ae3240b294dbfa93914f Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Thu, 31 Jan 2019 11:59:08 +0200 Subject: [PATCH 25/29] NY-6837 Remove old upload jobs - rebase after package refactoring; Signed-off-by: abotev-intracol --- .../nynja/content/db/configuration/CassandraConfig.java | 2 +- .../{JobScheduler.java => upload/rest/JobController.java} | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) rename src/main/java/biz/nynja/content/file/{JobScheduler.java => upload/rest/JobController.java} (87%) diff --git a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java index fd051ff..516887a 100644 --- a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java +++ b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java @@ -64,7 +64,7 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Override public String[] getEntityBasePackages() { - return new String[] { "biz.nynja.content.upload.models", "biz.nynja.content.file.metadata.dto" }; + return new String[] { "biz.nynja.content.file.upload.models", "biz.nynja.content.file.metadata.dto" }; } public String getConfiguredKeyspaceName() { diff --git a/src/main/java/biz/nynja/content/file/JobScheduler.java b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java similarity index 87% rename from src/main/java/biz/nynja/content/file/JobScheduler.java rename to src/main/java/biz/nynja/content/file/upload/rest/JobController.java index f2e8fea..f815a16 100644 --- a/src/main/java/biz/nynja/content/file/JobScheduler.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java @@ -1,4 +1,4 @@ -package biz.nynja.content.file; +package biz.nynja.content.file.upload.rest; import java.util.List; import java.util.Optional; @@ -11,17 +11,16 @@ import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.RestController; import biz.nynja.content.file.upload.models.UploadStore; -import biz.nynja.content.file.upload.rest.FileUploadController; import biz.nynja.content.file.upload.token.UploadTokenService; @RestController -public class JobScheduler { +public class JobController { private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); private UploadTokenService uploadTokenService; - public JobScheduler(UploadTokenService uploadTokenService) { + public JobController(UploadTokenService uploadTokenService) { this.uploadTokenService = uploadTokenService; } -- GitLab From d0aade5459bd955687f2e2c6c1a283ea1fffd594 Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Tue, 5 Feb 2019 17:04:23 +0200 Subject: [PATCH 26/29] Stop JWT authentication Signed-off-by: abotev-intracol --- .../templates/authentication-policy.yaml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/charts/content-service/templates/authentication-policy.yaml b/charts/content-service/templates/authentication-policy.yaml index df399c2..895af58 100644 --- a/charts/content-service/templates/authentication-policy.yaml +++ b/charts/content-service/templates/authentication-policy.yaml @@ -1,19 +1,19 @@ -apiVersion: "authentication.istio.io/v1alpha1" -kind: "Policy" -metadata: - name: {{ template "service.name" . }} - labels: - app: {{ template "service.name" . }} - chart: {{ template "service.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - targets: - - name: {{ template "service.name" . }} - origins: - - jwt: - issuer: https://auth.nynja.biz/ - jwksUri: http://auth-service.auth.svc.cluster.local:8008/keys/public - audiences: - - dGVzdEluc3RhbmNl:NynjaApp:NynjaOrg - principalBinding: USE_ORIGIN +#apiVersion: "authentication.istio.io/v1alpha1" +#kind: "Policy" +#metadata: +# name: {{ template "service.name" . }} +# labels: +# app: {{ template "service.name" . }} +# chart: {{ template "service.chart" . }} +# release: {{ .Release.Name }} +# heritage: {{ .Release.Service }} +#spec: +# targets: +# - name: {{ template "service.name" . }} +# origins: +# - jwt: +# issuer: https://auth.nynja.biz/ +# jwksUri: http://auth-service.auth.svc.cluster.local:8008/keys/public +# audiences: +# - dGVzdEluc3RhbmNl:NynjaApp:NynjaOrg +# principalBinding: USE_ORIGIN -- GitLab From acbc8d091d9293fd3418913e3a4cb66220b898fe Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 6 Feb 2019 16:12:57 +0200 Subject: [PATCH 27/29] NY-6843 Encode filename in DB - encode filename before write in DB; - decode filename when work with it; - fix file download to set right content type and filename; Signed-off-by: abotev-intracol --- .../file/download/FileDownloadController.java | 18 ++++++++++++++---- .../file/upload/rest/FileUploadService.java | 8 +++++--- .../file/upload/token/UploadTokenService.java | 14 ++++++++++++++ .../grpc/services/ContentServiceImpl.java | 5 +++-- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java index 5933218..0ea9039 100644 --- a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java +++ b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java @@ -12,6 +12,7 @@ import java.util.UUID; import javax.servlet.http.HttpServletRequest; +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -26,6 +27,8 @@ import org.springframework.web.bind.annotation.RestController; import com.nimbusds.jose.util.Base64URL; +import biz.nynja.content.file.upload.token.UploadTokenService; + /** * @author Angel.Botev * @@ -38,10 +41,12 @@ public class FileDownloadController { private static final Logger logger = LoggerFactory.getLogger(FileDownloadController.class); private FileDownloadService fileDownloadService; + private UploadTokenService uploadTokenService; @Autowired - public FileDownloadController(FileDownloadService fileDownloadService) { + public FileDownloadController(FileDownloadService fileDownloadService, UploadTokenService uploadTokenService) { this.fileDownloadService = fileDownloadService; + this.uploadTokenService = uploadTokenService; } @RequestMapping(method = RequestMethod.GET, value = "/{fileKey}") @@ -62,9 +67,11 @@ public class FileDownloadController { } // Try to determine file's content type String contentType = null; + String fileName = null; try { - contentType = request.getServletContext().getMimeType(resource.getFile().getAbsolutePath()); - } catch (IOException ex) { + fileName = uploadTokenService.decodeFileName(resource.getFile().getName()); + contentType = request.getServletContext().getMimeType(fileName); + } catch (Exception ex) { logger.info("Could not determine file type."); } @@ -72,10 +79,13 @@ public class FileDownloadController { if (contentType == null) { contentType = "application/octet-stream"; } + if (fileName == null) { + fileName = RandomStringUtils.randomAlphabetic(15); + } return ResponseEntity.ok().contentType(MediaType.parseMediaType(contentType)) .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + Base64URL.encode(resource.getFilename()) + "\"") + "attachment; filename=\"" + Base64URL.encode(fileName) + "\"") .body(resource); } } diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java index 1a235ef..f665d10 100644 --- a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -78,10 +78,12 @@ public class FileUploadService { } private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { - String fileName = constructFileName(uploadInfo.getFileName()); - uploadInfo.setFileName(fileName); + String fileName = uploadTokenService.decodeFileName(uploadInfo.getFileName()); + String constructedFilename = constructFileName(fileName); + String encodedFileName = uploadTokenService.encodeFileName(constructedFilename); + uploadInfo.setFileName(encodedFileName); - String uploadLocation = storageProvider.initialize(fileName); + String uploadLocation = storageProvider.initialize(encodedFileName); uploadInfo.setUploadLocation(uploadLocation); return uploadInfo; diff --git a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index c651136..f878c42 100644 --- a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -10,6 +10,7 @@ import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Optional; @@ -136,4 +137,17 @@ public class UploadTokenService { } return Optional.of(oldJobs); } + + public String encodeFileName(String fileName) { + String encodedFilename = Base64.getEncoder().encodeToString(fileName.getBytes()); + logger.debug("Filename: {} was encoded to: {}", fileName, encodedFilename); + return encodedFilename; + } + + public String decodeFileName(String encodedFilename) { + byte[] decodedFilenameInBytes = Base64.getDecoder().decode(encodedFilename); + String decodedFilename = new String(decodedFilenameInBytes); + logger.debug("Encoded filename: {} was decoded to: {}", encodedFilename, encodedFilename); + return decodedFilename; + } } diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 5713b2a..b301e5b 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -87,6 +87,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas request.getMediaType(), request.getSize(), request.getUploadedFrom(), request.getDeviceId()); String accountId = ContentServiceConstants.ACCOUNT_ID.get(); + String fileName = uploadTokenService.encodeFileName(request.getName()); if (StringUtils.isEmpty(accountId) || !util.isValidUuid(accountId)) { uploadTokenResponseProvider.prepareErrorResponse(responseObserver, Cause.INTERNAL_SERVER_ERROR, @@ -102,7 +103,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas validationResult.getErrorMessage()); return; } - UploadToken token = new UploadToken(request.getName(), request.getMediaType().name(), request.getSize(), + UploadToken token = new UploadToken(fileName, request.getMediaType().name(), request.getSize(), request.getUploadedFrom(), request.getDeviceId()); Optional tokenResponse = uploadTokenService.prepareTokenResponse(token); if (!tokenResponse.isPresent()) { @@ -115,7 +116,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas UploadStore uploadInfo = new UploadStore(tokenResponse.get(), Instant.now().toEpochMilli() + uploadTokenConfiguration.getTokenTimeToLive(), contentSreviceConfiguration.getUploadUrl() + jobId, jobId, request.getDeviceId(), - UUID.fromString(accountId), request.getName(), request.getSize(), request.getMediaType().name(), + UUID.fromString(accountId), fileName, request.getSize(), request.getMediaType().name(), request.getUploadedFrom(), UploadStatus.PENDING); if (!uploadTokenService.storeToken(uploadInfo, false)) { -- GitLab From 79d616457498a3c1bb00af063b03727d92ab3e1a Mon Sep 17 00:00:00 2001 From: abotev-intracol Date: Wed, 6 Feb 2019 18:08:59 +0200 Subject: [PATCH 28/29] NY-6843 Encode filename in DB - fix comments Signed-off-by: abotev-intracol --- .../biz/nynja/content/file/upload/token/UploadTokenService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index f878c42..df9487f 100644 --- a/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -147,7 +147,7 @@ public class UploadTokenService { public String decodeFileName(String encodedFilename) { byte[] decodedFilenameInBytes = Base64.getDecoder().decode(encodedFilename); String decodedFilename = new String(decodedFilenameInBytes); - logger.debug("Encoded filename: {} was decoded to: {}", encodedFilename, encodedFilename); + logger.debug("Encoded filename: {} was decoded to: {}", encodedFilename, decodedFilename); return decodedFilename; } } -- GitLab From 0001536c58747ba7dbf7222bd9a449ca63f45124 Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Wed, 20 Feb 2019 16:15:17 +0200 Subject: [PATCH 29/29] Add gcloud service account file. --- charts/content-service/templates/deployment.yaml | 10 ++++++++++ charts/content-service/values.yaml | 13 +++++++++++++ releases/dev/content-service.yaml | 13 +++++++++++++ 3 files changed, 36 insertions(+) diff --git a/charts/content-service/templates/deployment.yaml b/charts/content-service/templates/deployment.yaml index bf29d0b..090eeaf 100644 --- a/charts/content-service/templates/deployment.yaml +++ b/charts/content-service/templates/deployment.yaml @@ -58,6 +58,16 @@ spec: value: {{ .Values.ports.containerPort.http | quote }} - name: GRPC_SERVER_PORT value: {{ .Values.ports.containerPort.grpc | quote }} +{{ if .Values.extra_vars -}} +{{ toYaml .Values.extra_vars | indent 8 }} +{{- end }} + volumes: + - name: service-account + secret: + secretName: service-account + volumeMounts: + - name: service-account + mountPath: /opt/nynja/config resources: {{ toYaml .Values.resources | indent 12 }} {{- with .Values.nodeSelector }} diff --git a/charts/content-service/values.yaml b/charts/content-service/values.yaml index a1a770d..dc61f6f 100644 --- a/charts/content-service/values.yaml +++ b/charts/content-service/values.yaml @@ -36,3 +36,16 @@ corsPolicy: allowHeaders: maxAge: +extra_vars: + - name: FILE_UPLOAD_URL + value: https://content.dev-eu.nynja.net/file/upload + - name: FILE_DOWNLOAD_URL + value: https://content.dev-eu.nynja.net/file/download/ + - name: LOCAL_STORAGE_LOCATION + value: /src/main/resources + - name: GOOGLE_STORAGE_URI + value: https://storage.googleapis.com + - name: GOOGLE_STORAGE_BUCKET + value: content-service-dev + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /opt/nynja/application-credentials.json diff --git a/releases/dev/content-service.yaml b/releases/dev/content-service.yaml index 40887f1..745dc30 100644 --- a/releases/dev/content-service.yaml +++ b/releases/dev/content-service.yaml @@ -53,3 +53,16 @@ spec: - x-grpc-web maxAge: "600s" + extra_vars: + - name: FILE_UPLOAD_URL + value: https://content.dev-eu.nynja.net/file/upload + - name: FILE_DOWNLOAD_URL + value: https://content.dev-eu.nynja.net/file/download/ + - name: LOCAL_STORAGE_LOCATION + value: /src/main/resources + - name: GOOGLE_STORAGE_URI + value: https://storage.googleapis.com + - name: GOOGLE_STORAGE_BUCKET + value: content-service-dev + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /opt/nynja/config/application-credentials.json -- GitLab