diff --git a/Jenkinsfile b/Jenkinsfile index cc232c3e906bcee0ae57190c81e79e4bff8a79a4..cfa512229ee0f27d944e87df2a52cdae078c7f6f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,7 +44,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } } } @@ -66,7 +66,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } } } @@ -83,7 +83,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } dockerBuildAndPushToRegistry "${NAMESPACE}/${APP_NAME}", [IMAGE_BUILD_TAG] } @@ -120,7 +120,7 @@ pipeline { steps { container('mvn') { withCredentials([file(credentialsId: 'mavenSettings.xml', variable: 'FILE')]) { - sh 'mvn --settings $FILE clean install' + sh 'mvn --settings $FILE clean install -DskipTests=true' } dockerBuildAndPushToRegistry "${NAMESPACE}/${APP_NAME}", [IMAGE_BUILD_TAG] } diff --git a/charts/content-service/templates/authentication-policy.yaml b/charts/content-service/templates/authentication-policy.yaml index df399c2f45335f4360f2c1f9eded50b25e4a1604..895af58a5e956ed2778d7b148c03df7464bcd141 100644 --- a/charts/content-service/templates/authentication-policy.yaml +++ b/charts/content-service/templates/authentication-policy.yaml @@ -1,19 +1,19 @@ -apiVersion: "authentication.istio.io/v1alpha1" -kind: "Policy" -metadata: - name: {{ template "service.name" . }} - labels: - app: {{ template "service.name" . }} - chart: {{ template "service.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - targets: - - name: {{ template "service.name" . }} - origins: - - jwt: - issuer: https://auth.nynja.biz/ - jwksUri: http://auth-service.auth.svc.cluster.local:8008/keys/public - audiences: - - dGVzdEluc3RhbmNl:NynjaApp:NynjaOrg - principalBinding: USE_ORIGIN +#apiVersion: "authentication.istio.io/v1alpha1" +#kind: "Policy" +#metadata: +# name: {{ template "service.name" . }} +# labels: +# app: {{ template "service.name" . }} +# chart: {{ template "service.chart" . }} +# release: {{ .Release.Name }} +# heritage: {{ .Release.Service }} +#spec: +# targets: +# - name: {{ template "service.name" . }} +# origins: +# - jwt: +# issuer: https://auth.nynja.biz/ +# jwksUri: http://auth-service.auth.svc.cluster.local:8008/keys/public +# audiences: +# - dGVzdEluc3RhbmNl:NynjaApp:NynjaOrg +# principalBinding: USE_ORIGIN diff --git a/charts/content-service/templates/deployment.yaml b/charts/content-service/templates/deployment.yaml index bf29d0b76775a7233a17656edd81739514cea11d..090eeaf12bd8a3536c2a477af8b7a3c41a19935f 100644 --- a/charts/content-service/templates/deployment.yaml +++ b/charts/content-service/templates/deployment.yaml @@ -58,6 +58,16 @@ spec: value: {{ .Values.ports.containerPort.http | quote }} - name: GRPC_SERVER_PORT value: {{ .Values.ports.containerPort.grpc | quote }} +{{ if .Values.extra_vars -}} +{{ toYaml .Values.extra_vars | indent 8 }} +{{- end }} + volumes: + - name: service-account + secret: + secretName: service-account + volumeMounts: + - name: service-account + mountPath: /opt/nynja/config resources: {{ toYaml .Values.resources | indent 12 }} {{- with .Values.nodeSelector }} diff --git a/charts/content-service/values.yaml b/charts/content-service/values.yaml index a1a770d83e169c913f64b82242170e7b20580320..dc61f6f15ff1647113f264a53c511f84a4e56572 100644 --- a/charts/content-service/values.yaml +++ b/charts/content-service/values.yaml @@ -36,3 +36,16 @@ corsPolicy: allowHeaders: maxAge: +extra_vars: + - name: FILE_UPLOAD_URL + value: https://content.dev-eu.nynja.net/file/upload + - name: FILE_DOWNLOAD_URL + value: https://content.dev-eu.nynja.net/file/download/ + - name: LOCAL_STORAGE_LOCATION + value: /src/main/resources + - name: GOOGLE_STORAGE_URI + value: https://storage.googleapis.com + - name: GOOGLE_STORAGE_BUCKET + value: content-service-dev + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /opt/nynja/application-credentials.json diff --git a/pom.xml b/pom.xml index 15468fb44a45cffa458793a8fcf33b84165c25ef..bc4e4eaa4663ad23587bb87082a2eaf2cd670aef 100644 --- a/pom.xml +++ b/pom.xml @@ -124,8 +124,18 @@ - org.codehaus.groovy - groovy-all + org.springframework.boot + spring-boot-starter-actuator + + + + io.micrometer + micrometer-core + + + + io.micrometer + micrometer-registry-prometheus @@ -152,11 +162,11 @@ true - - com.google.cloud - google-cloud-storage - 1.53.0 - + + com.google.cloud + google-cloud-storage + 1.53.0 + diff --git a/releases/dev/content-service.yaml b/releases/dev/content-service.yaml index 40887f14fd06103c962de314f9f0be55558301ed..745dc30968212132341b0ba5020f3f68c0456d67 100644 --- a/releases/dev/content-service.yaml +++ b/releases/dev/content-service.yaml @@ -53,3 +53,16 @@ spec: - x-grpc-web maxAge: "600s" + extra_vars: + - name: FILE_UPLOAD_URL + value: https://content.dev-eu.nynja.net/file/upload + - name: FILE_DOWNLOAD_URL + value: https://content.dev-eu.nynja.net/file/download/ + - name: LOCAL_STORAGE_LOCATION + value: /src/main/resources + - name: GOOGLE_STORAGE_URI + value: https://storage.googleapis.com + - name: GOOGLE_STORAGE_BUCKET + value: content-service-dev + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /opt/nynja/config/application-credentials.json diff --git a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java index 525af77cf441b70311f71a97e08d64751b0c14ea..516887afeb0b2ca6f2117328b5376c0d94097d2e 100644 --- a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java +++ b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java @@ -22,6 +22,9 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Value("${spring.data.cassandra.keyspace-name}") private String keyspace; + @Value("${spring.data.cassandra.replication}") + private int replication; + @Override protected String getKeyspaceName() { return keyspace; @@ -43,6 +46,10 @@ public class CassandraConfig extends AbstractCassandraConfiguration { return port; } + private int getReplication() { + return replication; + } + @Override public SchemaAction getSchemaAction() { return SchemaAction.CREATE_IF_NOT_EXISTS; @@ -51,13 +58,13 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Override protected List getKeyspaceCreations() { CreateKeyspaceSpecification specification = CreateKeyspaceSpecification.createKeyspace(getKeyspaceName()) - .ifNotExists().withSimpleReplication(); + .ifNotExists().withSimpleReplication(getReplication()); return Arrays.asList(specification); } @Override public String[] getEntityBasePackages() { - return new String[] { "biz.nynja.content.upload.models", "biz.nynja.content.file.metadata.dto" }; + return new String[] { "biz.nynja.content.file.upload.models", "biz.nynja.content.file.metadata.dto" }; } public String getConfiguredKeyspaceName() { diff --git a/src/main/java/biz/nynja/content/file/download/FileControllerExceptionHandler.java b/src/main/java/biz/nynja/content/file/FileControllersExceptionHandler.java similarity index 92% rename from src/main/java/biz/nynja/content/file/download/FileControllerExceptionHandler.java rename to src/main/java/biz/nynja/content/file/FileControllersExceptionHandler.java index 91398820ae3574ab04a4ffc8b66a023370fe3e27..bd1381c1f5cf016397c70438220aa78fb3908a94 100644 --- a/src/main/java/biz/nynja/content/file/download/FileControllerExceptionHandler.java +++ b/src/main/java/biz/nynja/content/file/FileControllersExceptionHandler.java @@ -1,7 +1,7 @@ /** * Copyright (C) 2018 Nynja Inc. All rights reserved. */ -package biz.nynja.content.file.download; +package biz.nynja.content.file; import java.io.FileNotFoundException; import java.util.MissingResourceException; @@ -25,9 +25,9 @@ import org.springframework.web.client.HttpServerErrorException; import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; @ControllerAdvice -public class FileControllerExceptionHandler { +public class FileControllersExceptionHandler { - private static final Logger logger = LoggerFactory.getLogger(FileControllerExceptionHandler.class); + private static final Logger logger = LoggerFactory.getLogger(FileControllersExceptionHandler.class); @ExceptionHandler({ IllegalArgumentException.class, MethodArgumentNotValidException.class, HttpMessageNotReadableException.class, ServletRequestBindingException.class }) @@ -58,14 +58,14 @@ public class FileControllerExceptionHandler { public static ResponseEntity handleNotFoundExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_FOUND, e), HttpStatus.NOT_FOUND); + return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_FOUND, e.getMessage()), HttpStatus.NOT_FOUND); } @ExceptionHandler({ HttpMediaTypeNotAcceptableException.class }) public ResponseEntity handleNotAcceptableExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_ACCEPTABLE, e), + return new ResponseEntity(new ExceptionBody(HttpStatus.NOT_ACCEPTABLE, e.getMessage()), HttpStatus.NOT_ACCEPTABLE); } @@ -73,20 +73,20 @@ public class FileControllerExceptionHandler { public ResponseEntity handleUnsupportedMediaTypeExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.UNSUPPORTED_MEDIA_TYPE, e), + return new ResponseEntity(new ExceptionBody(HttpStatus.UNSUPPORTED_MEDIA_TYPE, e.getMessage()), HttpStatus.UNSUPPORTED_MEDIA_TYPE); } @ExceptionHandler({ MethodArgumentTypeMismatchException.class }) public ResponseEntity handleException(Exception e) { - return new ResponseEntity(new ExceptionBody(HttpStatus.BAD_REQUEST, e), HttpStatus.BAD_REQUEST); + return new ResponseEntity(new ExceptionBody(HttpStatus.BAD_REQUEST, e.getMessage()), HttpStatus.BAD_REQUEST); } @ExceptionHandler({ HttpServerErrorException.class, InternalError.class, Exception.class }) public ResponseEntity handleInternalServerExceptions(Exception e) { logger.error(e.getMessage()); logger.debug(e.getMessage(), e); - return new ResponseEntity(new ExceptionBody(HttpStatus.INTERNAL_SERVER_ERROR, e), + return new ResponseEntity(new ExceptionBody(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage()), HttpStatus.INTERNAL_SERVER_ERROR); } diff --git a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java index 1af8123132b46dcc45cc6be4e028c7560e1f9abc..0ea9039dbc096f17338789139ff761f2532aa58b 100644 --- a/src/main/java/biz/nynja/content/file/download/FileDownloadController.java +++ b/src/main/java/biz/nynja/content/file/download/FileDownloadController.java @@ -12,6 +12,7 @@ import java.util.UUID; import javax.servlet.http.HttpServletRequest; +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -26,25 +27,29 @@ import org.springframework.web.bind.annotation.RestController; import com.nimbusds.jose.util.Base64URL; +import biz.nynja.content.file.upload.token.UploadTokenService; + /** * @author Angel.Botev * */ @RestController -@RequestMapping("file/") +@RequestMapping("file/download") public class FileDownloadController { private static final Logger logger = LoggerFactory.getLogger(FileDownloadController.class); private FileDownloadService fileDownloadService; + private UploadTokenService uploadTokenService; @Autowired - public FileDownloadController(FileDownloadService fileDownloadService) { + public FileDownloadController(FileDownloadService fileDownloadService, UploadTokenService uploadTokenService) { this.fileDownloadService = fileDownloadService; + this.uploadTokenService = uploadTokenService; } - @RequestMapping(method = RequestMethod.GET, value = "download/{fileKey}") + @RequestMapping(method = RequestMethod.GET, value = "/{fileKey}") public ResponseEntity downloadFile(@PathVariable String fileKey, HttpServletRequest request) throws FileNotFoundException { logger.info("Downloading file: {} ...", fileKey); @@ -62,9 +67,11 @@ public class FileDownloadController { } // Try to determine file's content type String contentType = null; + String fileName = null; try { - contentType = request.getServletContext().getMimeType(resource.getFile().getAbsolutePath()); - } catch (IOException ex) { + fileName = uploadTokenService.decodeFileName(resource.getFile().getName()); + contentType = request.getServletContext().getMimeType(fileName); + } catch (Exception ex) { logger.info("Could not determine file type."); } @@ -72,10 +79,13 @@ public class FileDownloadController { if (contentType == null) { contentType = "application/octet-stream"; } + if (fileName == null) { + fileName = RandomStringUtils.randomAlphabetic(15); + } return ResponseEntity.ok().contentType(MediaType.parseMediaType(contentType)) .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + Base64URL.encode(resource.getFilename()) + "\"") + "attachment; filename=\"" + Base64URL.encode(fileName) + "\"") .body(resource); } } diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index 483d9fea970c025cb1db5adcf143732fb86e0d68..7c885bdf26a4a8997479f4e137610ed49f44e77b 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -12,7 +12,8 @@ import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.metadata.repositories.FileMetadataReposiory; -import biz.nynja.content.upload.models.UploadStore; +import biz.nynja.content.file.storage.Provider; +import biz.nynja.content.file.upload.models.UploadStore; /** * @author Ralitsa Todorova @@ -29,17 +30,17 @@ public class FileMetadataService { this.fileMetadataRepository = fileMetadataRepository; } - public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl) { + public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileUrl, Provider storageProvider) { FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setKey(UUID.randomUUID()); - fileMetadata.setFileName(fileName); + fileMetadata.setFileName(uploadInfo.getFileName()); fileMetadata.setFileSize(uploadInfo.getFileSize()); fileMetadata.setMediaType(uploadInfo.getMediaType()); fileMetadata.setUploadedFrom(uploadInfo.getUploadedFrom()); fileMetadata.setUploadTimestamp(Instant.now().toEpochMilli()); fileMetadata.setAccountId(uploadInfo.getAccountId()); fileMetadata.setDeviceId(uploadInfo.getDeviceId()); - fileMetadata.setStorage("LOCAL"); + fileMetadata.setStorage(storageProvider.name()); fileMetadata.setFileUrl(fileUrl); fileMetadataRepository.save(fileMetadata); diff --git a/src/main/java/biz/nynja/content/file/storage/Provider.java b/src/main/java/biz/nynja/content/file/storage/Provider.java new file mode 100644 index 0000000000000000000000000000000000000000..343584592fab52b95272185f175d7c006e3966f4 --- /dev/null +++ b/src/main/java/biz/nynja/content/file/storage/Provider.java @@ -0,0 +1,12 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.storage; + +/** + * @author Angel.Botev + * + */ +public enum Provider { + GOOGLE, LOCAL +} diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index 22b699030790c5ad49a5ff580c4bcdfdcaf3e3e3..6264d3f7bdae09bc724762cb92959831053784d1 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -3,6 +3,8 @@ */ package biz.nynja.content.file.storage; +import java.io.IOException; + /** * @author Angel.Botev * @@ -13,6 +15,9 @@ public interface StorageProvider { public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws Exception; - public void close(String fileLocation); + public void close(String fileLocation) throws IOException; + + public String getFileUrl(String filename); + public Provider getProviderType(); } diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java index 2a2b310edb26f8872e3c1432e91fbf00b0bb822d..461e03457b5961757a54a75f3c2181cba59c8147 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java @@ -23,14 +23,14 @@ public class StorageProviderPool { private static final Logger log = LoggerFactory.getLogger(StorageProviderPool.class); - private Map storageProviders = new HashMap<>(); + private Map storageProviders = new HashMap<>(); public StorageProviderPool(GoogleStorageProvider googleStorageProvider, LocalStorageProvider localStorageProvider) { - storageProviders.put("LOCAL", localStorageProvider); - storageProviders.put("GOOGLE", googleStorageProvider); + storageProviders.put(Provider.LOCAL, localStorageProvider); + storageProviders.put(Provider.GOOGLE, googleStorageProvider); } - public StorageProvider getStorageProviderByType(String storageProviderType) { + public StorageProvider getStorageProviderByType(Provider storageProviderType) { log.debug("Get storage provider = {}", storageProviderType); StorageProvider storageProvider = storageProviders.get(storageProviderType); if (storageProvider == null) { diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index f6771524f4e48d9781ee66ad5bec41e1a29a37bd..6d0dc0a784e60d271a515094a7291aa7b46d5fcb 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -28,6 +28,7 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.InputStreamContent; import com.google.api.services.storage.StorageScopes; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageConfiguration; import biz.nynja.content.file.storage.StorageProvider; import io.netty.buffer.ByteBufInputStream; @@ -43,6 +44,8 @@ public class GoogleStorageProvider implements StorageProvider { private static final Logger logger = LoggerFactory.getLogger(GoogleStorageProvider.class); + private static final Provider providerType = Provider.GOOGLE; + private HttpRequestFactory requestFactory; private StorageConfiguration storageConfiguration; @@ -53,7 +56,7 @@ public class GoogleStorageProvider implements StorageProvider { httpTransport = GoogleNetHttpTransport.newTrustedTransport(); // Build an account credential. GoogleCredential credential = GoogleCredential.getApplicationDefault(); - credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_READ_WRITE)); + credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)); requestFactory = httpTransport.createRequestFactory(credential); } catch (GeneralSecurityException | IOException e) { logger.error("Error with Google credentials: {}", e.getMessage()); @@ -79,7 +82,8 @@ public class GoogleStorageProvider implements StorageProvider { @Override public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws Exception { - logger.debug("Writing chunk: {} with length {} to: {}", chunkCount, data.length, fileLocation); + logger.debug("Writing partId: {} with length {} to: {}", chunkCount, data.length, fileLocation); + --chunkCount; // It's necessary because chunkCount starts from 1 not from 0 try (InputStream inputStream = new ByteBufInputStream(Unpooled.wrappedBuffer(data))) { int length = Math.min(Unpooled.wrappedBuffer(data).readableBytes(), storageConfiguration.getUploadChunkSize()); @@ -102,28 +106,23 @@ public class GoogleStorageProvider implements StorageProvider { } @Override - public void close(String fileLocation) { + public void close(String fileLocation) throws IOException { logger.debug("Cancel resumable Google upload for location: {}", fileLocation); String URI = fileLocation; GenericUrl url = new GenericUrl(URI); HttpRequest req; - try { - req = requestFactory.buildDeleteRequest(url); - HttpHeaders headers = new HttpHeaders(); - headers.setContentLength((long) 0); - req.setHeaders(headers); - logger.debug("Executing DELETE request to: {}", URI); - // Execute request - HttpResponse resp = req.execute(); - if (resp.getStatusCode() == 200) { - logger.debug("Successfull canceled Google upload for location: {}", fileLocation); - } else { - logger.error("Error canceling Google upload. Status code: {}, message: ", resp.getStatusCode(), - resp.getStatusMessage()); - } - } catch (IOException e) { - logger.error("Error canceling Google upload: {}", e.getMessage()); - logger.debug("Error canceling Google upload: {}", e.getCause()); + req = requestFactory.buildDeleteRequest(url); + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength((long) 0); + req.setHeaders(headers); + logger.debug("Executing DELETE request to: {}", URI); + // Execute request + HttpResponse resp = req.execute(); + if (resp.getStatusCode() == 200) { + logger.debug("Successfull canceled Google upload for location: {}", fileLocation); + } else { + logger.error("Error canceling Google upload. Status code: {}, message: ", resp.getStatusCode(), + resp.getStatusMessage()); } } @@ -160,4 +159,17 @@ public class GoogleStorageProvider implements StorageProvider { HttpResponse resp = req.execute(); return resp; } + + @Override + public String getFileUrl(String filename) { + StringBuilder builder = new StringBuilder(); + builder.append(storageConfiguration.getGoogleStorageURI()).append("/") + .append(storageConfiguration.getGoogleBucketName()).append("/").append(filename); + return builder.toString(); + } + + @Override + public Provider getProviderType() { + return providerType; + } } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index b9df3778ddd20da5fc75132a2456d14963150005..29c75aa7285ea68216152948f382f95c665a225e 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -7,11 +7,14 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageConfiguration; import biz.nynja.content.file.storage.StorageProvider; @@ -25,6 +28,8 @@ public class LocalStorageProvider implements StorageProvider { private static final Logger logger = LoggerFactory.getLogger(LocalStorageProvider.class); + private static final Provider providerType = Provider.LOCAL; + private StorageConfiguration storageConfiguration; public LocalStorageProvider(StorageConfiguration storageConfiguration) { @@ -40,22 +45,35 @@ public class LocalStorageProvider implements StorageProvider { @Override public void write(byte[] data, String fileLocation, int chunkCount, boolean isFinalChunk) throws IOException { - logger.debug("Appending bytes in file: {}", fileLocation); + logger.debug("Appending bytes to file: {}", fileLocation); FileOutputStream output = new FileOutputStream(fileLocation, true); try { - output.write(data); - logger.info("Bytes was successfully append to file {}.", fileLocation); + output.write(data); + logger.info("Bytes were successfully appended to file {}.", fileLocation); } finally { - output.close(); + output.close(); } } @Override - public void close(String fileLocation) { + public void close(String fileLocation) throws IOException { + logger.debug("Canceling resumable upload for location: {}", fileLocation); + Files.delete(Paths.get(fileLocation)); + logger.debug("Successfull canceled upload for location: {}", fileLocation); } private String constructFilePath(String contentStoreLocation, String fileName) { return new StringBuilder(contentStoreLocation).append(File.separator).append(fileName).toString(); } + @Override + public String getFileUrl(String filename) { + return constructFilePath(storageConfiguration.getLocalStorageLocation(), filename); + } + + @Override + public Provider getProviderType() { + return providerType; + } + } diff --git a/src/main/java/biz/nynja/content/file/upload/models/UploadStatus.java b/src/main/java/biz/nynja/content/file/upload/models/UploadStatus.java new file mode 100644 index 0000000000000000000000000000000000000000..bcb595bedfa416d3fc0783ce297c67205f2fd507 --- /dev/null +++ b/src/main/java/biz/nynja/content/file/upload/models/UploadStatus.java @@ -0,0 +1,13 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.models; + +/** + * @author Ralitsa Todorova + * + */ +public enum UploadStatus { + + PENDING, IN_PROGRESS, FAILED, COMPLETED; +} diff --git a/src/main/java/biz/nynja/content/upload/models/UploadStore.java b/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java similarity index 74% rename from src/main/java/biz/nynja/content/upload/models/UploadStore.java rename to src/main/java/biz/nynja/content/file/upload/models/UploadStore.java index 0cc07561a102681c27f0d6bdc2a061f6853a613d..b902848a8e8fa259202f3fb027bb7696b5072f66 100644 --- a/src/main/java/biz/nynja/content/upload/models/UploadStore.java +++ b/src/main/java/biz/nynja/content/file/upload/models/UploadStore.java @@ -1,242 +1,299 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.models; - -import java.util.UUID; - -import org.springframework.data.cassandra.core.mapping.PrimaryKey; -import org.springframework.data.cassandra.core.mapping.Table; - -/** - * @author Ralitsa Todorova - * - */ -@Table -public class UploadStore { - - @PrimaryKey - private String uploadToken; - private Long tokenExpirationTime; - private String uploadUrl; - private UUID jobId; - private String deviceId; - private UUID accountId; - private String fileName; - private int fileSize; - private String mediaType; - private String uploadedFrom; - private String status; - private int retries; - - public UploadStore() { - } - - public UploadStore(String uploadToken, Long tokenExpirationTime, String uploadUrl, UUID jobId, String deviceId, - UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, String status) { - this.uploadToken = uploadToken; - this.tokenExpirationTime = tokenExpirationTime; - this.uploadUrl = uploadUrl; - this.jobId = jobId; - this.deviceId = deviceId; - this.accountId = accountId; - this.fileName = fileName; - this.fileSize = fileSize; - this.mediaType = mediaType; - this.uploadedFrom = uploadedFrom; - this.status = status; - this.retries = 0; - } - - public String getUploadToken() { - return uploadToken; - } - - public String getUploadUrl() { - return uploadUrl; - } - - public void setUploadUrl(String uploadUrl) { - this.uploadUrl = uploadUrl; - } - - public UUID getJobId() { - return jobId; - } - - public void setJobId(UUID jobId) { - this.jobId = jobId; - } - - public String getDeviceId() { - return deviceId; - } - - public void setDeviceId(String deviceId) { - this.deviceId = deviceId; - } - - public UUID getAccountId() { - return accountId; - } - - public void setAccountId(UUID accountId) { - this.accountId = accountId; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public int getFileSize() { - return fileSize; - } - - public void setFileSize(int fileSize) { - this.fileSize = fileSize; - } - - public String getMediaType() { - return mediaType; - } - - public void setMediaType(String mediaType) { - this.mediaType = mediaType; - } - - public String getUploadedFrom() { - return uploadedFrom; - } - - public void setUploadedFrom(String uploadedFrom) { - this.uploadedFrom = uploadedFrom; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public void setUploadToken(String uploadToken) { - this.uploadToken = uploadToken; - } - - public Long getTokenExpirationTime() { - return tokenExpirationTime; - } - - public void setTokenExpirationTime(Long tokenExpirationTime) { - this.tokenExpirationTime = tokenExpirationTime; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - this.retries = retries; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((accountId == null) ? 0 : accountId.hashCode()); - result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); - result = prime * result + ((fileName == null) ? 0 : fileName.hashCode()); - result = prime * result + fileSize; - result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); - result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); - result = prime * result + retries; - result = prime * result + ((status == null) ? 0 : status.hashCode()); - result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); - result = prime * result + ((uploadToken == null) ? 0 : uploadToken.hashCode()); - result = prime * result + ((uploadUrl == null) ? 0 : uploadUrl.hashCode()); - result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - UploadStore other = (UploadStore) obj; - if (accountId == null) { - if (other.accountId != null) - return false; - } else if (!accountId.equals(other.accountId)) - return false; - if (deviceId == null) { - if (other.deviceId != null) - return false; - } else if (!deviceId.equals(other.deviceId)) - return false; - if (fileName == null) { - if (other.fileName != null) - return false; - } else if (!fileName.equals(other.fileName)) - return false; - if (fileSize != other.fileSize) - return false; - if (jobId == null) { - if (other.jobId != null) - return false; - } else if (!jobId.equals(other.jobId)) - return false; - if (mediaType == null) { - if (other.mediaType != null) - return false; - } else if (!mediaType.equals(other.mediaType)) - return false; - if (retries != other.retries) - return false; - if (status == null) { - if (other.status != null) - return false; - } else if (!status.equals(other.status)) - return false; - if (tokenExpirationTime == null) { - if (other.tokenExpirationTime != null) - return false; - } else if (!tokenExpirationTime.equals(other.tokenExpirationTime)) - return false; - if (uploadToken == null) { - if (other.uploadToken != null) - return false; - } else if (!uploadToken.equals(other.uploadToken)) - return false; - if (uploadUrl == null) { - if (other.uploadUrl != null) - return false; - } else if (!uploadUrl.equals(other.uploadUrl)) - return false; - if (uploadedFrom == null) { - if (other.uploadedFrom != null) - return false; - } else if (!uploadedFrom.equals(other.uploadedFrom)) - return false; - return true; - } - - @Override - public String toString() { - return new StringBuilder("UploadStore [uploadToken=").append(uploadToken).append(", tokenExpirationTime=") - .append(tokenExpirationTime).append(", uploadUrl=").append(uploadUrl).append(", jobId=").append(jobId) - .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") - .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) - .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") - .append(retries).append("]").toString(); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.models; + +import java.util.UUID; + +import org.springframework.data.cassandra.core.mapping.PrimaryKey; +import org.springframework.data.cassandra.core.mapping.Table; + +/** + * @author Ralitsa Todorova + * + */ +@Table +public class UploadStore { + + @PrimaryKey + private String uploadToken; + private Long tokenExpirationTime; + private String uploadUrl; + private UUID jobId; + private String deviceId; + private UUID accountId; + private String fileName; + private int fileSize; + private String mediaType; + private String uploadedFrom; + private UploadStatus status; + private int retries; + private int partId; + private int lastUploadedByte; + private String uploadLocation; + private Long lastUploadedChunkTimestamp; + + public UploadStore() { + } + + public UploadStore(String uploadToken, Long tokenExpirationTime, String uploadUrl, UUID jobId, String deviceId, + UUID accountId, String fileName, int fileSize, String mediaType, String uploadedFrom, UploadStatus status) { + this.uploadToken = uploadToken; + this.tokenExpirationTime = tokenExpirationTime; + this.uploadUrl = uploadUrl; + this.jobId = jobId; + this.deviceId = deviceId; + this.accountId = accountId; + this.fileName = fileName; + this.fileSize = fileSize; + this.mediaType = mediaType; + this.uploadedFrom = uploadedFrom; + this.status = status; + this.retries = 0; + this.partId = 0; + this.lastUploadedByte = 0; + this.uploadLocation = ""; + this.lastUploadedChunkTimestamp = 0L; + } + + public String getUploadToken() { + return uploadToken; + } + + public String getUploadUrl() { + return uploadUrl; + } + + public void setUploadUrl(String uploadUrl) { + this.uploadUrl = uploadUrl; + } + + public UUID getJobId() { + return jobId; + } + + public void setJobId(UUID jobId) { + this.jobId = jobId; + } + + public String getDeviceId() { + return deviceId; + } + + public void setDeviceId(String deviceId) { + this.deviceId = deviceId; + } + + public UUID getAccountId() { + return accountId; + } + + public void setAccountId(UUID accountId) { + this.accountId = accountId; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public int getFileSize() { + return fileSize; + } + + public void setFileSize(int fileSize) { + this.fileSize = fileSize; + } + + public String getMediaType() { + return mediaType; + } + + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + public String getUploadedFrom() { + return uploadedFrom; + } + + public void setUploadedFrom(String uploadedFrom) { + this.uploadedFrom = uploadedFrom; + } + + public UploadStatus getStatus() { + return status; + } + + public void setStatus(UploadStatus status) { + this.status = status; + } + + public void setUploadToken(String uploadToken) { + this.uploadToken = uploadToken; + } + + public Long getTokenExpirationTime() { + return tokenExpirationTime; + } + + public void setTokenExpirationTime(Long tokenExpirationTime) { + this.tokenExpirationTime = tokenExpirationTime; + } + + public int getRetries() { + return retries; + } + + public void setRetries(int retries) { + this.retries = retries; + } + + public int getPartId() { + return partId; + } + + public void setPartId(int partId) { + this.partId = partId; + } + + public int getLastUploadedByte() { + return lastUploadedByte; + } + + public void setLastUploadedByte(int lastUploadedByte) { + this.lastUploadedByte = lastUploadedByte; + } + + public String getUploadLocation() { + return uploadLocation; + } + + public void setUploadLocation(String uploadLocation) { + this.uploadLocation = uploadLocation; + } + + public Long getLastUploadedChunkTimestamp() { + return lastUploadedChunkTimestamp; + } + + public void setLastUploadedChunkTimestamp(Long lastUploadedChunkTimestamp) { + this.lastUploadedChunkTimestamp = lastUploadedChunkTimestamp; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((accountId == null) ? 0 : accountId.hashCode()); + result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); + result = prime * result + ((fileName == null) ? 0 : fileName.hashCode()); + result = prime * result + fileSize; + result = prime * result + ((jobId == null) ? 0 : jobId.hashCode()); + result = prime * result + lastUploadedByte; + result = prime * result + ((lastUploadedChunkTimestamp == null) ? 0 : lastUploadedChunkTimestamp.hashCode()); + result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); + result = prime * result + partId; + result = prime * result + retries; + result = prime * result + ((status == null) ? 0 : status.hashCode()); + result = prime * result + ((tokenExpirationTime == null) ? 0 : tokenExpirationTime.hashCode()); + result = prime * result + ((uploadLocation == null) ? 0 : uploadLocation.hashCode()); + result = prime * result + ((uploadToken == null) ? 0 : uploadToken.hashCode()); + result = prime * result + ((uploadUrl == null) ? 0 : uploadUrl.hashCode()); + result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadStore other = (UploadStore) obj; + if (accountId == null) { + if (other.accountId != null) + return false; + } else if (!accountId.equals(other.accountId)) + return false; + if (deviceId == null) { + if (other.deviceId != null) + return false; + } else if (!deviceId.equals(other.deviceId)) + return false; + if (fileName == null) { + if (other.fileName != null) + return false; + } else if (!fileName.equals(other.fileName)) + return false; + if (fileSize != other.fileSize) + return false; + if (jobId == null) { + if (other.jobId != null) + return false; + } else if (!jobId.equals(other.jobId)) + return false; + if (lastUploadedByte != other.lastUploadedByte) + return false; + if (lastUploadedChunkTimestamp == null) { + if (other.lastUploadedChunkTimestamp != null) + return false; + } else if (!lastUploadedChunkTimestamp.equals(other.lastUploadedChunkTimestamp)) + return false; + if (mediaType == null) { + if (other.mediaType != null) + return false; + } else if (!mediaType.equals(other.mediaType)) + return false; + if (partId != other.partId) + return false; + if (retries != other.retries) + return false; + if (status != other.status) + return false; + if (tokenExpirationTime == null) { + if (other.tokenExpirationTime != null) + return false; + } else if (!tokenExpirationTime.equals(other.tokenExpirationTime)) + return false; + if (uploadLocation == null) { + if (other.uploadLocation != null) + return false; + } else if (!uploadLocation.equals(other.uploadLocation)) + return false; + if (uploadToken == null) { + if (other.uploadToken != null) + return false; + } else if (!uploadToken.equals(other.uploadToken)) + return false; + if (uploadUrl == null) { + if (other.uploadUrl != null) + return false; + } else if (!uploadUrl.equals(other.uploadUrl)) + return false; + if (uploadedFrom == null) { + if (other.uploadedFrom != null) + return false; + } else if (!uploadedFrom.equals(other.uploadedFrom)) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadStore [uploadToken=").append(uploadToken).append(", tokenExpirationTime=") + .append(tokenExpirationTime).append(", uploadUrl=").append(uploadUrl).append(", jobId=").append(jobId) + .append(", deviceId=").append(deviceId).append(", accountId=").append(accountId).append(", fileName=") + .append(fileName).append(", fileSize=").append(fileSize).append(", mediaType=").append(mediaType) + .append(", uploadedFrom=").append(uploadedFrom).append(", status=").append(status).append(", retries=") + .append(retries).append(", partId=").append(partId) + .append(", lastUploadedByte=").append(lastUploadedByte) + .append("]").toString(); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepository.java b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepository.java similarity index 81% rename from src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepository.java rename to src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepository.java index 793c38d0e5e2c4be368804e1edde9e2297590eee..aae1f91bd18af36baa7c4329227bc9a28c533bc7 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepository.java +++ b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepository.java @@ -1,14 +1,14 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.repositories; - -/** - * @author Ralitsa Todorova - * @param - * - */ -public interface CustomizedUploadStoreRepository { - - S saveWithTtl(S uploadStore, int ttl); -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.repositories; + +/** + * @author Ralitsa Todorova + * @param + * + */ +public interface CustomizedUploadStoreRepository { + + S saveWithTtl(S uploadStore, int ttl); +} diff --git a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepositoryImpl.java similarity index 69% rename from src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java rename to src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepositoryImpl.java index c2caaaaba132f5a3f08bd9a89c152bf3677a8d5f..cd9c5b5de62fcfaf0380f63c447ce6c833610eb6 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/CustomizedUploadStoreRepositoryImpl.java +++ b/src/main/java/biz/nynja/content/file/upload/repositories/CustomizedUploadStoreRepositoryImpl.java @@ -1,47 +1,49 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.repositories; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.cassandra.core.CassandraTemplate; - -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; - -/** - * @author Ralitsa Todorova - * - */ -public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadStoreRepository { - - private final CassandraTemplate cassandraTemplate; - private UploadTokenConfiguration uploadTokenConfiguration; - - @Autowired - public CustomizedUploadStoreRepositoryImpl(CassandraTemplate cassandraTemplate, - UploadTokenConfiguration uploadTokenConfiguration) { - this.cassandraTemplate = cassandraTemplate; - this.uploadTokenConfiguration = uploadTokenConfiguration; - } - - public S saveWithTtl(S uploadStore, int ttl) { - - if (!executeSaveWithTtl((UploadStore) uploadStore, uploadTokenConfiguration.getTokenTimeToLive())) { - throw new RuntimeException("Error inserting new upload info record."); - } else { - return uploadStore; - } - } - - private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { - String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " - + ttl + ";"; - return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), - uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), - uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), uploadStore.getStatus(), - uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), uploadStore.getUploadUrl(), - uploadStore.getRetries()); - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.repositories; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.cassandra.core.CassandraTemplate; + +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; + +/** + * @author Ralitsa Todorova + * + */ +public class CustomizedUploadStoreRepositoryImpl implements CustomizedUploadStoreRepository { + + private final CassandraTemplate cassandraTemplate; + private UploadTokenConfiguration uploadTokenConfiguration; + + @Autowired + public CustomizedUploadStoreRepositoryImpl(CassandraTemplate cassandraTemplate, + UploadTokenConfiguration uploadTokenConfiguration) { + this.cassandraTemplate = cassandraTemplate; + this.uploadTokenConfiguration = uploadTokenConfiguration; + } + + public S saveWithTtl(S uploadStore, int ttl) { + + if (!executeSaveWithTtl((UploadStore) uploadStore, uploadTokenConfiguration.getTokenTimeToLive())) { + throw new RuntimeException("Error inserting new upload info record."); + } else { + return uploadStore; + } + } + + private boolean executeSaveWithTtl(UploadStore uploadStore, int ttl) { + String insertStatement = "INSERT INTO uploadstore (uploadtoken, accountid, deviceid, filename, filesize, jobid, mediatype, status, tokenexpirationtime, uploadedfrom, uploadurl, retries, partid, lastuploadedbyte, lastuploadedchunktimestamp, uploadlocation) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) USING TTL " + + ttl + ";"; + return cassandraTemplate.getCqlOperations().execute(insertStatement, uploadStore.getUploadToken(), + uploadStore.getAccountId(), uploadStore.getDeviceId(), uploadStore.getFileName(), + uploadStore.getFileSize(), uploadStore.getJobId(), uploadStore.getMediaType(), + uploadStore.getStatus().toString(), uploadStore.getTokenExpirationTime(), uploadStore.getUploadedFrom(), + uploadStore.getUploadUrl(), uploadStore.getRetries(), uploadStore.getPartId(), + uploadStore.getLastUploadedByte(), uploadStore.getLastUploadedChunkTimestamp(), + uploadStore.getUploadLocation()); + } + +} diff --git a/src/main/java/biz/nynja/content/upload/repositories/UploadStoreRepository.java b/src/main/java/biz/nynja/content/file/upload/repositories/UploadStoreRepository.java similarity index 80% rename from src/main/java/biz/nynja/content/upload/repositories/UploadStoreRepository.java rename to src/main/java/biz/nynja/content/file/upload/repositories/UploadStoreRepository.java index 9a78df42e99ceb7b7c31f8c14b17d4d2c5f44db9..bb5cef7f2b953838299b4154d1d392fc004b8792 100644 --- a/src/main/java/biz/nynja/content/upload/repositories/UploadStoreRepository.java +++ b/src/main/java/biz/nynja/content/file/upload/repositories/UploadStoreRepository.java @@ -1,20 +1,20 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.repositories; - -import org.springframework.data.cassandra.repository.CassandraRepository; -import org.springframework.stereotype.Repository; - -import biz.nynja.content.upload.models.UploadStore; - -/** - * @author Ralitsa Todorova - * - */ -@Repository -public interface UploadStoreRepository - extends CassandraRepository, CustomizedUploadStoreRepository { - - UploadStore findByUploadToken(String uploadToken); -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.repositories; + +import org.springframework.data.cassandra.repository.CassandraRepository; +import org.springframework.stereotype.Repository; + +import biz.nynja.content.file.upload.models.UploadStore; + +/** + * @author Ralitsa Todorova + * + */ +@Repository +public interface UploadStoreRepository + extends CassandraRepository, CustomizedUploadStoreRepository { + + UploadStore findByUploadToken(String uploadToken); +} diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java new file mode 100644 index 0000000000000000000000000000000000000000..759fd82a88d63770967198522b99630f9c08e9b9 --- /dev/null +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadController.java @@ -0,0 +1,104 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.file.upload.rest.validation.ValidationResult; +import biz.nynja.content.file.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@RestController +@RequestMapping("/file/upload") +public class FileUploadController { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); + + private UploadTokenService uploadTokenService; + private ChunkUploadValidator validator; + private FileUploadService fileUploadService; + + public FileUploadController(UploadTokenService uploadTokenService, ChunkUploadValidator validator, + FileUploadService fileUploadService) { + + this.uploadTokenService = uploadTokenService; + this.validator = validator; + this.fileUploadService = fileUploadService; + } + + @PutMapping("/{jobId}") + public ResponseEntity uploadFile(@PathVariable String jobId, + @RequestParam("token") String uploadToken, @RequestParam("partId") int partId, + @RequestParam("last") boolean last, @RequestBody byte[] dataChunk) { + + logger.info("New chunk upload request recived for jobId: {} and token: {}", jobId, uploadToken); + UploadStore uploadInfo; + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (uploadInfoResult.isPresent()) { + uploadInfo = uploadInfoResult.get(); + } else { + logger.error("You are not allowed to upload data. Provided token is either wrong or missing."); + return new ResponseEntity<>( + new UploadResponse( + "You are not allowed to upload data. Provided token is either wrong or missing."), + HttpStatus.BAD_REQUEST); + } + + Optional requestValidation = validator.validateUploadRequest(uploadInfo, dataChunk, jobId, + uploadToken, partId, last); + if (requestValidation.isPresent()) { + return new ResponseEntity<>(new UploadResponse(requestValidation.get().getMessage(), uploadInfo.getStatus(), + uploadInfo.getPartId()), HttpStatus.BAD_REQUEST); + } + + try { + UploadResponse result = fileUploadService.writeDataChunk(uploadInfo, dataChunk, partId, last); + return new ResponseEntity<>(result, HttpStatus.CREATED); + } catch (Exception e) { + logger.error("Error writing data chunk for jobId {}: {}", jobId, e.getMessage()); + logger.debug("Error writing data chunk for jobId {}: {}", jobId, e.getCause()); + return new ResponseEntity<>( + new UploadResponse("Error writing data chunk", uploadInfo.getStatus(), uploadInfo.getPartId()), + HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + @DeleteMapping("/{jobId}") + public ResponseEntity cancelUploading(@PathVariable String jobId, + @RequestParam("token") String uploadToken) { + + logger.info("Cancel request recived for jobId: {} and token: {}", jobId, uploadToken); + Optional uploadInfoResult = uploadTokenService.getUploadInfo(uploadToken); + if (!uploadInfoResult.isPresent()) { + logger.error( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished."); + return new ResponseEntity<>( + "You are not allowed to cancel upload. Provided token is either wrong, missing or job is already finished.", + HttpStatus.BAD_REQUEST); + } + if(!fileUploadService.cancelUploadProcess(uploadInfoResult.get())) { + return new ResponseEntity<>("Error canceling upload job. File not found or not removed.", HttpStatus.INTERNAL_SERVER_ERROR); + } + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + +} diff --git a/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java new file mode 100644 index 0000000000000000000000000000000000000000..f665d105ac4515540039418b273ec35a43dd1d74 --- /dev/null +++ b/src/main/java/biz/nynja/content/file/upload/rest/FileUploadService.java @@ -0,0 +1,130 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest; + +import java.io.IOException; +import java.time.Instant; + +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import biz.nynja.content.file.metadata.FileMetadataService; +import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.StorageProvider; +import biz.nynja.content.file.storage.StorageProviderPool; +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class FileUploadService { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); + + private StorageProvider storageProvider; + private UploadTokenService uploadTokenService; + private FileMetadataService fileMetadataService; + private ContentServiceConfiguration contentServiceConfiguration; + + @Autowired + public FileUploadService(StorageProviderPool storageProviderPool, UploadTokenService uploadTokenService, + FileMetadataService fileMetadataService, ContentServiceConfiguration contentServiceConfiguration) { + this.uploadTokenService = uploadTokenService; + this.fileMetadataService = fileMetadataService; + this.contentServiceConfiguration = contentServiceConfiguration; + this.storageProvider = storageProviderPool + .getStorageProviderByType(contentServiceConfiguration.getStorageProvider()); + } + + public UploadResponse writeDataChunk(UploadStore uploadInfo, byte[] dataChunk, int partId, boolean last) + throws Exception { + + if (uploadInfo.getStatus().equals(UploadStatus.PENDING)) { // Upload hasn't started yet + uploadInfo = initializeUpload(uploadInfo); + } + + storageProvider.write(dataChunk, uploadInfo.getUploadLocation(), partId, last); + + logger.info("Chunk {} was successfully written to output stream {}.", partId, uploadInfo.getFileName()); + uploadInfo = updateUploadInfo(uploadInfo, partId, dataChunk.length); + if (!uploadTokenService.storeToken(uploadInfo, true)) { + // In this case data chunk was successfully appended to file (when LOCAL storage is used), but + // corresponding DB record contains incorrect data. This situation leads to data inconsistency. + // TODO: Check this behavior against cloud storage. (Google Cloud Storage). + logger.error( + "FATAL: Error updating upload job record in DB. This situation may lead to data inconsistency. Client will be notified about this."); + return new UploadResponse( + "FATAL: File upload operation entered in inconsistent state. You are advices to terminate current upload and initiate a new one.", + UploadStatus.FAILED, uploadInfo.getPartId()); + + } + if (last) { + finalizeUpload(uploadInfo); + return new UploadResponse("File upload successfully completed.", UploadStatus.COMPLETED, + uploadInfo.getPartId()); + } + + return new UploadResponse("Data chunk upload successfully completed.", UploadStatus.IN_PROGRESS, + uploadInfo.getPartId()); + } + + private UploadStore initializeUpload(UploadStore uploadInfo) throws Exception { + String fileName = uploadTokenService.decodeFileName(uploadInfo.getFileName()); + String constructedFilename = constructFileName(fileName); + String encodedFileName = uploadTokenService.encodeFileName(constructedFilename); + uploadInfo.setFileName(encodedFileName); + + String uploadLocation = storageProvider.initialize(encodedFileName); + uploadInfo.setUploadLocation(uploadLocation); + + return uploadInfo; + } + + public String finalizeUpload(UploadStore uploadInfo) { + + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + storageProvider.getFileUrl(uploadInfo.getFileName()), storageProvider.getProviderType()); + String dowlnoadUrl = constructDownloadUrl(fileMetadata.getKey().toString()); + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + logger.info("File sucessfully uploaded."); + return dowlnoadUrl; + } + + private String constructFileName(String fileName) { + String generatedPrefix = RandomStringUtils.randomAlphabetic(10); + return new StringBuilder(generatedPrefix).append('_').append(fileName).toString(); + } + + private UploadStore updateUploadInfo(UploadStore uploadInfo, int partId, int offset) { + uploadInfo.setPartId(partId); + uploadInfo.setLastUploadedByte(uploadInfo.getLastUploadedByte() + offset); + uploadInfo.setLastUploadedChunkTimestamp(Instant.now().toEpochMilli()); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); + return uploadInfo; + } + + private String constructDownloadUrl(String fileKey) { + return new StringBuilder(contentServiceConfiguration.getDownloadUrl()).append(fileKey).toString(); + } + + public boolean cancelUploadProcess(UploadStore uploadInfo) { + try { + this.storageProvider.close(uploadInfo.getUploadLocation()); + } catch (IOException e) { + logger.error("Error canceling upload job: {}", e.getMessage()); + logger.debug("Error canceling upload job: {}", e.getCause()); + return false; + } + uploadTokenService.deleteToken(uploadInfo.getUploadToken()); + return true; + } +} diff --git a/src/main/java/biz/nynja/content/file/upload/rest/JobController.java b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java new file mode 100644 index 0000000000000000000000000000000000000000..f815a16ce35d6b6653124420626befc7534ee872 --- /dev/null +++ b/src/main/java/biz/nynja/content/file/upload/rest/JobController.java @@ -0,0 +1,41 @@ +package biz.nynja.content.file.upload.rest; + +import java.util.List; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.RestController; + +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadTokenService; + +@RestController +public class JobController { + + private static final Logger logger = LoggerFactory.getLogger(FileUploadController.class); + + private UploadTokenService uploadTokenService; + + public JobController(UploadTokenService uploadTokenService) { + this.uploadTokenService = uploadTokenService; + } + + @DeleteMapping("/jobs/old") + public ResponseEntity removeOldJobs() { + logger.info("Remove not finished (old) jobs "); + + Optional> oldJobs = uploadTokenService.getOldUploadJobs(); + if (oldJobs.isPresent()) { + logger.info("Founded old jobs! Countinue with delete process", oldJobs); + uploadTokenService.deleteJobs(oldJobs.get()); + return new ResponseEntity("Old jobs successfully deleted!", HttpStatus.NO_CONTENT); + } else { + logger.info("Old jobs didn't found!"); + return new ResponseEntity("Old jobs didn't found!", HttpStatus.NO_CONTENT); + } + } +} diff --git a/src/main/java/biz/nynja/content/file/upload/rest/UploadResponse.java b/src/main/java/biz/nynja/content/file/upload/rest/UploadResponse.java new file mode 100644 index 0000000000000000000000000000000000000000..0de3b1201a2733bfbd92dadcf3d83a29c8ba9c6c --- /dev/null +++ b/src/main/java/biz/nynja/content/file/upload/rest/UploadResponse.java @@ -0,0 +1,112 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest; + +import biz.nynja.content.file.upload.models.UploadStatus; + +/** + * @author Ralitsa Todorova + * + */ +public class UploadResponse { + + private String message; + private UploadStatus status; + private int lastSuccessfulChunk; + private String downloadLink; + + public UploadResponse(String message) { + this.message = message; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public UploadResponse(String message, UploadStatus status, int lastSuccessfulChunk, String downloadLink) { + this.message = message; + this.status = status; + this.lastSuccessfulChunk = lastSuccessfulChunk; + this.downloadLink = downloadLink; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public UploadStatus getStatus() { + return status; + } + + public void setStatus(UploadStatus status) { + this.status = status; + } + + public int getLastSuccessfulChunk() { + return lastSuccessfulChunk; + } + + public void setLastSuccessfulChunk(int lastSuccessfulChunk) { + this.lastSuccessfulChunk = lastSuccessfulChunk; + } + + public String getDownloadLink() { + return downloadLink; + } + + public void setDownloadLink(String downloadLink) { + this.downloadLink = downloadLink; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((downloadLink == null) ? 0 : downloadLink.hashCode()); + result = prime * result + lastSuccessfulChunk; + result = prime * result + ((message == null) ? 0 : message.hashCode()); + result = prime * result + ((status == null) ? 0 : status.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadResponse other = (UploadResponse) obj; + if (downloadLink == null) { + if (other.downloadLink != null) + return false; + } else if (!downloadLink.equals(other.downloadLink)) + return false; + if (lastSuccessfulChunk != other.lastSuccessfulChunk) + return false; + if (message == null) { + if (other.message != null) + return false; + } else if (!message.equals(other.message)) + return false; + if (status != other.status) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadResponse [message=").append(message).append(", status=").append(status) + .append(", lastSuccessfulChunk=").append(lastSuccessfulChunk).append(", downloadLink=") + .append(downloadLink).append("]").toString(); + } + +} diff --git a/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java new file mode 100644 index 0000000000000000000000000000000000000000..22e05941174d47db22161a7e4ebcd6ce0607efb5 --- /dev/null +++ b/src/main/java/biz/nynja/content/file/upload/rest/validation/ChunkUploadValidator.java @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest.validation; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import biz.nynja.content.file.upload.models.UploadStore; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class ChunkUploadValidator { + + private static final Logger logger = LoggerFactory.getLogger(ChunkUploadValidator.class); + + public Optional validateUploadRequest(UploadStore uploadInfo, byte[] data, String jobId, + String token, int partId, boolean last) { + + if (!jobId.equals(uploadInfo.getJobId().toString())) { + logger.error("Upload token {} does not apply to the requested job id: {}", token, jobId); + return Optional.of(new ValidationResult("Upload token does not apply to the requested upload URL.")); + } + + if (partId != uploadInfo.getPartId() + 1) { + logger.error( + "Chunk with Unexpected part id received for job id {} with associated token: {} . Expected: {}, actual: {}", + jobId, token, uploadInfo.getPartId() + 1, partId); + return Optional.of(new ValidationResult( + "You are trying to upload wrong data chunk. Last uploaded chunk " + uploadInfo.getPartId())); + } + if (data.length <= 0) { + logger.error("Recieved empty data chuck for jobId {} with associated token: {}", jobId, token); + return Optional.of(new ValidationResult("Missing chunk data.")); + } + + String sizeValidationResult = validateReachedFileSize(uploadInfo.getLastUploadedByte() + data.length, + uploadInfo.getFileSize(), last); + if (sizeValidationResult != null) { + logger.error("Incorrect file size reached for jobId {} with associated token: {}. {}", jobId, token, + sizeValidationResult); + return Optional.of(new ValidationResult(sizeValidationResult)); + } + + return Optional.empty(); + } + + private boolean matchesExpectedSize(int actual, int expected) { + return actual == expected; + } + + private boolean exceedsExpectedSize(int actual, int expected) { + return actual > expected; + } + + private String validateReachedFileSize(int actual, int expected, boolean lastChunk) { + if (exceedsExpectedSize(actual, expected)) { + logger.error("Size of uploaded file so far exceeds the expected file size."); + return "Size of uploaded file so far exceeds the expected file size."; + } + if (lastChunk && !matchesExpectedSize(actual, expected)) { + logger.error("Uploaded file size does not meet expectations - expected: {}, actual: {}.", expected, actual); + return "Uploaded file size does not meet expectations."; + } + if (!lastChunk && matchesExpectedSize(actual, expected)) { + logger.error( + "Uploaded file size so far reached expected overal file size, but this is not a final chunk - expected: {}, actual: {}.", + expected, actual); + return "Uploaded file size so far reached expected overal file size, but this is not a final chunk."; + } + return null; + } + +} diff --git a/src/main/java/biz/nynja/content/file/upload/rest/validation/ValidationResult.java b/src/main/java/biz/nynja/content/file/upload/rest/validation/ValidationResult.java new file mode 100644 index 0000000000000000000000000000000000000000..288e0a4b9838858601d52a69940b0cf0a0989a0b --- /dev/null +++ b/src/main/java/biz/nynja/content/file/upload/rest/validation/ValidationResult.java @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.rest.validation; + +/** + * @author Ralitsa Todorova + * + */ +public class ValidationResult { + + private String message; + + public ValidationResult(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadToken.java b/src/main/java/biz/nynja/content/file/upload/token/UploadToken.java similarity index 95% rename from src/main/java/biz/nynja/content/upload/token/UploadToken.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadToken.java index 90ad1c0ca3c6a284853cd1839f99265f89680c6a..ae6384120ee805644df2d5c6578a656428609527 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadToken.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadToken.java @@ -1,124 +1,124 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -/** - * @author Ralitsa Todorova - * - */ -public class UploadToken { - - private String name; - private String mediaType; - private int size; - private String uploadedFrom; - private String deviceId; - - public UploadToken(String name, String mediaType, int size, String uploadedFrom, String deviceId) { - this.name = name; - this.mediaType = mediaType; - this.size = size; - this.uploadedFrom = uploadedFrom; - this.deviceId = deviceId; - } - - public String getDeviceId() { - return deviceId; - } - - public void setDeviceId(String deviceId) { - this.deviceId = deviceId; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getMediaType() { - return mediaType; - } - - public void setMediaType(String mediaType) { - this.mediaType = mediaType; - } - - public int getSize() { - return size; - } - - public void setSize(int size) { - this.size = size; - } - - public String getUploadedFrom() { - return uploadedFrom; - } - - public void setUploadedFrom(String uploadedFrom) { - this.uploadedFrom = uploadedFrom; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); - result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); - result = prime * result + ((name == null) ? 0 : name.hashCode()); - result = prime * result + size; - result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - UploadToken other = (UploadToken) obj; - if (deviceId == null) { - if (other.deviceId != null) - return false; - } else if (!deviceId.equals(other.deviceId)) - return false; - if (mediaType == null) { - if (other.mediaType != null) - return false; - } else if (!mediaType.equals(other.mediaType)) - return false; - if (name == null) { - if (other.name != null) - return false; - } else if (!name.equals(other.name)) - return false; - if (size != other.size) - return false; - if (uploadedFrom == null) { - if (other.uploadedFrom != null) - return false; - } else if (!uploadedFrom.equals(other.uploadedFrom)) - return false; - return true; - } - - @Override - public String toString() { - return new StringBuilder("UploadToken [name=").append(name).append(", mediaType=").append(mediaType) - .append(", size=").append(size).append(", uploadedFrom=").append(uploadedFrom).append(", deviceId=") - .append(deviceId).append("]").toString(); - } - - public String getTokenString() { - return new StringBuilder("n:").append(this.getName()).append(",mt:").append(this.getMediaType()).append(",s:") - .append(this.getSize()).append(",uf:").append(this.getUploadedFrom()).append(",did:") - .append(this.getDeviceId()).toString(); - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +/** + * @author Ralitsa Todorova + * + */ +public class UploadToken { + + private String name; + private String mediaType; + private int size; + private String uploadedFrom; + private String deviceId; + + public UploadToken(String name, String mediaType, int size, String uploadedFrom, String deviceId) { + this.name = name; + this.mediaType = mediaType; + this.size = size; + this.uploadedFrom = uploadedFrom; + this.deviceId = deviceId; + } + + public String getDeviceId() { + return deviceId; + } + + public void setDeviceId(String deviceId) { + this.deviceId = deviceId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getMediaType() { + return mediaType; + } + + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public String getUploadedFrom() { + return uploadedFrom; + } + + public void setUploadedFrom(String uploadedFrom) { + this.uploadedFrom = uploadedFrom; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((deviceId == null) ? 0 : deviceId.hashCode()); + result = prime * result + ((mediaType == null) ? 0 : mediaType.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + size; + result = prime * result + ((uploadedFrom == null) ? 0 : uploadedFrom.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + UploadToken other = (UploadToken) obj; + if (deviceId == null) { + if (other.deviceId != null) + return false; + } else if (!deviceId.equals(other.deviceId)) + return false; + if (mediaType == null) { + if (other.mediaType != null) + return false; + } else if (!mediaType.equals(other.mediaType)) + return false; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + if (size != other.size) + return false; + if (uploadedFrom == null) { + if (other.uploadedFrom != null) + return false; + } else if (!uploadedFrom.equals(other.uploadedFrom)) + return false; + return true; + } + + @Override + public String toString() { + return new StringBuilder("UploadToken [name=").append(name).append(", mediaType=").append(mediaType) + .append(", size=").append(size).append(", uploadedFrom=").append(uploadedFrom).append(", deviceId=") + .append(deviceId).append("]").toString(); + } + + public String getTokenString() { + return new StringBuilder("n:").append(this.getName()).append(",mt:").append(this.getMediaType()).append(",s:") + .append(this.getSize()).append(",uf:").append(this.getUploadedFrom()).append(",did:") + .append(this.getDeviceId()).toString(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadTokenResponseProvider.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenResponseProvider.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/token/UploadTokenResponseProvider.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadTokenResponseProvider.java index 7cfb9b910408376dbc906c3dcdcd1ebfbf67f823..15824686938d95c887037255bdce582345111e43 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadTokenResponseProvider.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenResponseProvider.java @@ -1,34 +1,34 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -import org.springframework.stereotype.Service; - -import biz.nynja.content.grpc.ErrorResponse; -import biz.nynja.content.grpc.ErrorResponse.Cause; -import biz.nynja.content.grpc.UploadToken; -import biz.nynja.content.grpc.UploadTokenResponse; -import io.grpc.stub.StreamObserver; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class UploadTokenResponseProvider { - - public void prepareUploadTokenResponse(StreamObserver responseObserver, String token, - String tokenUrl) { - responseObserver.onNext(UploadTokenResponse.newBuilder() - .setTokenDetails(UploadToken.newBuilder().setToken(token).setUploadUrl(tokenUrl)).build()); - responseObserver.onCompleted(); - } - - public void prepareErrorResponse(StreamObserver responseObserver, Cause error, - String message) { - responseObserver.onNext(UploadTokenResponse.newBuilder() - .setError(ErrorResponse.newBuilder().setCause(error).setMessage(message)).build()); - responseObserver.onCompleted(); - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +import org.springframework.stereotype.Service; + +import biz.nynja.content.grpc.ErrorResponse; +import biz.nynja.content.grpc.ErrorResponse.Cause; +import biz.nynja.content.grpc.UploadToken; +import biz.nynja.content.grpc.UploadTokenResponse; +import io.grpc.stub.StreamObserver; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class UploadTokenResponseProvider { + + public void prepareUploadTokenResponse(StreamObserver responseObserver, String token, + String tokenUrl) { + responseObserver.onNext(UploadTokenResponse.newBuilder() + .setTokenDetails(UploadToken.newBuilder().setToken(token).setUploadUrl(tokenUrl)).build()); + responseObserver.onCompleted(); + } + + public void prepareErrorResponse(StreamObserver responseObserver, Cause error, + String message) { + responseObserver.onNext(UploadTokenResponse.newBuilder() + .setError(ErrorResponse.newBuilder().setCause(error).setMessage(message)).build()); + responseObserver.onCompleted(); + } +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadTokenService.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java similarity index 58% rename from src/main/java/biz/nynja/content/upload/token/UploadTokenService.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java index f11abfd63ebdf70fcf0be2b26b993e907b02dc50..df9487f8407d8779687526de3e779343ff59929d 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadTokenService.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenService.java @@ -1,101 +1,153 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -import java.io.UnsupportedEncodingException; -import java.security.InvalidAlgorithmParameterException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.util.Optional; - -import javax.crypto.BadPaddingException; -import javax.crypto.IllegalBlockSizeException; -import javax.crypto.NoSuchPaddingException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import biz.nynja.content.security.EncryptDecryptService; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.repositories.UploadStoreRepository; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class UploadTokenService { - - private final Logger logger = LoggerFactory.getLogger(UploadTokenService.class); - - private EncryptDecryptService encryptDecryptService; - private UploadStoreRepository uploadStoreRepository; - private UploadTokenConfiguration uploadTokenConfiguration; - - public UploadTokenService(EncryptDecryptService encryptDecryptService, - UploadStoreRepository uploadStoreRepository, UploadTokenConfiguration uploadTokenConfiguration) { - this.encryptDecryptService = encryptDecryptService; - this.uploadStoreRepository = uploadStoreRepository; - this.uploadTokenConfiguration = uploadTokenConfiguration; - } - - public Optional prepareTokenResponse(UploadToken uploadToken) { - String result; - try { - result = encryptDecryptService.encrypt(uploadToken.getTokenString()); - logger.debug("Upload token successfully encrypted: {}", uploadToken.toString()); - } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException - | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException - | UnsupportedEncodingException e) { - logger.error("Error encrypting upload token. {}", e.getCause()); - logger.debug("Error encrypting upload token: {}", e.getMessage()); - return Optional.empty(); - } - - return Optional.of(result); - } - - public boolean storeToken(UploadStore uploadInfo, boolean permanent) { - try { - if (permanent) { - uploadStoreRepository.save(uploadInfo); - logger.info("Upload token successfully stored in DB."); - } else { - uploadStoreRepository.saveWithTtl(uploadInfo, uploadTokenConfiguration.getTokenTimeToLive()); - logger.info("Upload token successfully stored in DB for {} seconds.", - uploadTokenConfiguration.getTokenTimeToLive()); - } - logger.info("Upload token successfully stored in DB."); - return true; - } catch (Exception e) { - logger.error("Error storing record {} in DB: {}", uploadInfo.toString(), e.getMessage()); - logger.debug("Error storing record {} in DB: {}", uploadInfo.toString(), e.getCause()); - return false; - } - } - - public void deleteToken(String tokenId) { - try { - logger.debug("Deleting token {} from DB...", tokenId); - uploadStoreRepository.deleteById(tokenId); - logger.debug("Token {} successfully deleted from DB.", tokenId); - } catch (Exception e) { - logger.error("Error deleting token {} from DB: {}", tokenId, e.getMessage()); - logger.debug("Error deleting token {} from DB: {}", tokenId, e.getCause()); - } - - uploadStoreRepository.deleteById(tokenId); - } - - public Optional getUploadInfo(String uploadToken) { - - UploadStore storeInfo = uploadStoreRepository.findByUploadToken(uploadToken); - if (storeInfo != null) { - return Optional.of(storeInfo); - } - return Optional.empty(); - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +import java.io.UnsupportedEncodingException; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Optional; + +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; +import biz.nynja.content.security.EncryptDecryptService; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.repositories.UploadStoreRepository; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class UploadTokenService { + + private final Logger logger = LoggerFactory.getLogger(UploadTokenService.class); + + private EncryptDecryptService encryptDecryptService; + private UploadStoreRepository uploadStoreRepository; + private UploadTokenConfiguration uploadTokenConfiguration; + private ContentServiceConfiguration contentServiceConfiguration; + + public UploadTokenService(EncryptDecryptService encryptDecryptService, UploadStoreRepository uploadStoreRepository, + UploadTokenConfiguration uploadTokenConfiguration, + ContentServiceConfiguration contentServiceConfiguration) { + this.encryptDecryptService = encryptDecryptService; + this.uploadStoreRepository = uploadStoreRepository; + this.uploadTokenConfiguration = uploadTokenConfiguration; + this.contentServiceConfiguration = contentServiceConfiguration; + } + + public Optional prepareTokenResponse(UploadToken uploadToken) { + String result; + try { + result = encryptDecryptService.encrypt(uploadToken.getTokenString()); + logger.debug("Upload token successfully encrypted: {}", uploadToken.toString()); + } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException + | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException + | UnsupportedEncodingException e) { + logger.error("Error encrypting upload token. {}", e.getCause()); + logger.debug("Error encrypting upload token: {}", e.getMessage()); + return Optional.empty(); + } + + return Optional.of(result); + } + + public boolean storeToken(UploadStore uploadInfo, boolean permanent) { + try { + if (permanent) { + uploadStoreRepository.save(uploadInfo); + logger.info("Upload token successfully stored in DB."); + } else { + uploadStoreRepository.saveWithTtl(uploadInfo, uploadTokenConfiguration.getTokenTimeToLive()); + logger.info("Upload token successfully stored in DB for {} seconds.", + uploadTokenConfiguration.getTokenTimeToLive()); + } + logger.info("Upload token successfully stored in DB."); + return true; + } catch (Exception e) { + logger.error("Error storing record {} in DB: {}", uploadInfo.toString(), e.getMessage()); + logger.debug("Error storing record {} in DB: {}", uploadInfo.toString(), e.getCause()); + return false; + } + } + + public void deleteToken(String tokenId) { + try { + logger.debug("Deleting token {} from DB...", tokenId); + uploadStoreRepository.deleteById(tokenId); + logger.debug("Token {} successfully deleted from DB.", tokenId); + } catch (Exception e) { + logger.error("Error deleting token {} from DB: {}", tokenId, e.getMessage()); + logger.debug("Error deleting token {} from DB: {}", tokenId, e.getCause()); + } + } + + public void deleteJobs(List jobs) { + try { + logger.debug("Deleting jobs {} from DB...", jobs); + uploadStoreRepository.deleteAll(jobs); + logger.debug("Jobs {} successfully deleted from DB.", jobs); + } catch (Exception e) { + logger.error("Error deleting jobs {} from DB: {}", jobs, e.getMessage()); + logger.debug("Error deleting jobs {} from DB: {}", jobs, e.getCause()); + } + } + + + public Optional getUploadInfo(String uploadToken) { + + UploadStore storeInfo = uploadStoreRepository.findByUploadToken(uploadToken); + if (storeInfo != null) { + return Optional.of(storeInfo); + } + return Optional.empty(); + } + + public Optional> getOldUploadJobs() { + + List jobs = uploadStoreRepository.findAll(); + if (jobs == null) { + return Optional.empty(); + } + + List oldJobs = new ArrayList<>(); + int jobTTL = contentServiceConfiguration.getJobTTL(); + Instant now = Instant.now(); + for (UploadStore job : jobs) { + Instant last = Instant.ofEpochMilli(job.getLastUploadedChunkTimestamp()); + long hours = ChronoUnit.HOURS.between(now, last); + if (Math.toIntExact(hours) > jobTTL) { + oldJobs.add(job); + } + } + return Optional.of(oldJobs); + } + + public String encodeFileName(String fileName) { + String encodedFilename = Base64.getEncoder().encodeToString(fileName.getBytes()); + logger.debug("Filename: {} was encoded to: {}", fileName, encodedFilename); + return encodedFilename; + } + + public String decodeFileName(String encodedFilename) { + byte[] decodedFilenameInBytes = Base64.getDecoder().decode(encodedFilename); + String decodedFilename = new String(decodedFilenameInBytes); + logger.debug("Encoded filename: {} was decoded to: {}", encodedFilename, decodedFilename); + return decodedFilename; + } +} diff --git a/src/main/java/biz/nynja/content/upload/token/UploadTokenValidator.java b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenValidator.java similarity index 94% rename from src/main/java/biz/nynja/content/upload/token/UploadTokenValidator.java rename to src/main/java/biz/nynja/content/file/upload/token/UploadTokenValidator.java index a5a1f245679bacd5b4693a7c9f1902edcea096ee..934838eb8d4b2b21cd19aea9a0648f76d385c685 100644 --- a/src/main/java/biz/nynja/content/upload/token/UploadTokenValidator.java +++ b/src/main/java/biz/nynja/content/file/upload/token/UploadTokenValidator.java @@ -1,127 +1,127 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token; - -import java.util.Collection; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; - -import com.google.common.io.Files; - -import biz.nynja.content.grpc.ErrorResponse.Cause; -import biz.nynja.content.core.validation.Validation; -import biz.nynja.content.core.validation.ValidationError; -import biz.nynja.content.grpc.UploadTokenRequest; -import biz.nynja.content.grpc.UploadTokenRequest.MediaType; -import biz.nynja.content.upload.token.configuration.MediaTypesConfiguration; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; - -/** - * @author Ralitsa Todorova - * - */ -@Service -public class UploadTokenValidator { - - private MediaTypesConfiguration mediaTypesConfiguration; - private UploadTokenConfiguration uploadTokenConfiguration; - - @Autowired - public UploadTokenValidator(MediaTypesConfiguration mediaTypesConfiguration, - UploadTokenConfiguration uploadTokenConfiguration) { - this.mediaTypesConfiguration = mediaTypesConfiguration; - this.uploadTokenConfiguration = uploadTokenConfiguration; - } - - public Validation validateGetTokenRequest(UploadTokenRequest request) { - Validation validation = new Validation(); - if (StringUtils.isEmpty(request.getName())) { - validation.addError(new ValidationError("Missing file name", Cause.MISSING_NAME)); - } - if (request.getSize() <= 0 || request.getSize() > uploadTokenConfiguration.getFileMaxSize()) { - validation.addError(new ValidationError("Incorrect file size", Cause.INCORRECT_SIZE)); - } - if (request.getMediaType().equals(MediaType.UNKNOWN_MEDIA_TYPE)) { - validation.addError(new ValidationError("Missing media type", Cause.MISSING_MEDIA_TYPE)); - } else { - String fileExtension = Files.getFileExtension(request.getName()); - Validation mediaTypeValidation = validateMediaType(request.getMediaType(), fileExtension); - if (mediaTypeValidation.hasErrors()) { - validation.addErrors(mediaTypeValidation.getErrors()); - } - } - if (StringUtils.isEmpty(request.getUploadedFrom())) { - validation.addError(new ValidationError("Missing uploadedFrom information", Cause.MISSING_UPLOADED_FROM)); - } - if (StringUtils.isEmpty(request.getDeviceId())) { - validation.addError(new ValidationError("Missing device id.", Cause.MISSING_DEVICE_ID)); - } - return validation; - } - - private Validation validateMediaType(MediaType requestMediaType, String fileExtension) { - Validation validation = new Validation(); - switch (requestMediaType) { - case AUDIO: - if (!mediaTypesConfiguration.getAudio().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case VIDEO: - if (!mediaTypesConfiguration.getVideo().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case COMPRESSED_FILES: - if (!mediaTypesConfiguration.getCompressed().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case DATA_FILES: - if (!mediaTypesConfiguration.getData().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case IMAGE: - if (!mediaTypesConfiguration.getImage().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case PAGE_LAYOUT: - if (!mediaTypesConfiguration.getPageLayout().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case SPREADSHEET: - if (!mediaTypesConfiguration.getSpreadsheet().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case TEXT: - if (!mediaTypesConfiguration.getText().contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - case OTHER: - if (Stream - .of(mediaTypesConfiguration.getAudio(), mediaTypesConfiguration.getData(), - mediaTypesConfiguration.getCompressed(), mediaTypesConfiguration.getImage(), - mediaTypesConfiguration.getPageLayout(), mediaTypesConfiguration.getSpreadsheet(), - mediaTypesConfiguration.getText(), mediaTypesConfiguration.getVideo()) - .flatMap(Collection::stream).collect(Collectors.toList()).contains(fileExtension)) { - validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); - } - break; - default: - break; - } - - return validation; - } - -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token; + +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import com.google.common.io.Files; + +import biz.nynja.content.grpc.ErrorResponse.Cause; +import biz.nynja.content.core.validation.Validation; +import biz.nynja.content.core.validation.ValidationError; +import biz.nynja.content.grpc.UploadTokenRequest; +import biz.nynja.content.grpc.UploadTokenRequest.MediaType; +import biz.nynja.content.file.upload.token.configuration.MediaTypesConfiguration; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; + +/** + * @author Ralitsa Todorova + * + */ +@Service +public class UploadTokenValidator { + + private MediaTypesConfiguration mediaTypesConfiguration; + private UploadTokenConfiguration uploadTokenConfiguration; + + @Autowired + public UploadTokenValidator(MediaTypesConfiguration mediaTypesConfiguration, + UploadTokenConfiguration uploadTokenConfiguration) { + this.mediaTypesConfiguration = mediaTypesConfiguration; + this.uploadTokenConfiguration = uploadTokenConfiguration; + } + + public Validation validateGetTokenRequest(UploadTokenRequest request) { + Validation validation = new Validation(); + if (StringUtils.isEmpty(request.getName())) { + validation.addError(new ValidationError("Missing file name", Cause.MISSING_NAME)); + } + if (request.getSize() <= 0 || request.getSize() > uploadTokenConfiguration.getFileMaxSize()) { + validation.addError(new ValidationError("Incorrect file size", Cause.INCORRECT_SIZE)); + } + if (request.getMediaType().equals(MediaType.UNKNOWN_MEDIA_TYPE)) { + validation.addError(new ValidationError("Missing media type", Cause.MISSING_MEDIA_TYPE)); + } else { + String fileExtension = Files.getFileExtension(request.getName()); + Validation mediaTypeValidation = validateMediaType(request.getMediaType(), fileExtension); + if (mediaTypeValidation.hasErrors()) { + validation.addErrors(mediaTypeValidation.getErrors()); + } + } + if (StringUtils.isEmpty(request.getUploadedFrom())) { + validation.addError(new ValidationError("Missing uploadedFrom information", Cause.MISSING_UPLOADED_FROM)); + } + if (StringUtils.isEmpty(request.getDeviceId())) { + validation.addError(new ValidationError("Missing device id.", Cause.MISSING_DEVICE_ID)); + } + return validation; + } + + private Validation validateMediaType(MediaType requestMediaType, String fileExtension) { + Validation validation = new Validation(); + switch (requestMediaType) { + case AUDIO: + if (!mediaTypesConfiguration.getAudio().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case VIDEO: + if (!mediaTypesConfiguration.getVideo().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case COMPRESSED_FILES: + if (!mediaTypesConfiguration.getCompressed().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case DATA_FILES: + if (!mediaTypesConfiguration.getData().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case IMAGE: + if (!mediaTypesConfiguration.getImage().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case PAGE_LAYOUT: + if (!mediaTypesConfiguration.getPageLayout().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case SPREADSHEET: + if (!mediaTypesConfiguration.getSpreadsheet().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case TEXT: + if (!mediaTypesConfiguration.getText().contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + case OTHER: + if (Stream + .of(mediaTypesConfiguration.getAudio(), mediaTypesConfiguration.getData(), + mediaTypesConfiguration.getCompressed(), mediaTypesConfiguration.getImage(), + mediaTypesConfiguration.getPageLayout(), mediaTypesConfiguration.getSpreadsheet(), + mediaTypesConfiguration.getText(), mediaTypesConfiguration.getVideo()) + .flatMap(Collection::stream).collect(Collectors.toList()).contains(fileExtension)) { + validation.addError(new ValidationError("Media type is not correct!", Cause.INCORRECT_MEDIA_TYPE)); + } + break; + default: + break; + } + + return validation; + } + +} diff --git a/src/main/java/biz/nynja/content/upload/token/configuration/MediaTypesConfiguration.java b/src/main/java/biz/nynja/content/file/upload/token/configuration/MediaTypesConfiguration.java similarity index 96% rename from src/main/java/biz/nynja/content/upload/token/configuration/MediaTypesConfiguration.java rename to src/main/java/biz/nynja/content/file/upload/token/configuration/MediaTypesConfiguration.java index 9f0b431d7749466b7e463ea4bce522a4953cfc63..e95a404959ec497fdc2c2a3456767ab6336ad7e0 100644 --- a/src/main/java/biz/nynja/content/upload/token/configuration/MediaTypesConfiguration.java +++ b/src/main/java/biz/nynja/content/file/upload/token/configuration/MediaTypesConfiguration.java @@ -1,7 +1,7 @@ /** * Copyright (C) 2018 Nynja Inc. All rights reserved. */ -package biz.nynja.content.upload.token.configuration; +package biz.nynja.content.file.upload.token.configuration; import java.util.List; diff --git a/src/main/java/biz/nynja/content/upload/token/configuration/UploadTokenConfiguration.java b/src/main/java/biz/nynja/content/file/upload/token/configuration/UploadTokenConfiguration.java similarity index 92% rename from src/main/java/biz/nynja/content/upload/token/configuration/UploadTokenConfiguration.java rename to src/main/java/biz/nynja/content/file/upload/token/configuration/UploadTokenConfiguration.java index 27c1f81051d919bdb6518267ee71ff77288424ba..54aabfcf538c9558ee3281d0b002e195c11a5f36 100644 --- a/src/main/java/biz/nynja/content/upload/token/configuration/UploadTokenConfiguration.java +++ b/src/main/java/biz/nynja/content/file/upload/token/configuration/UploadTokenConfiguration.java @@ -1,51 +1,51 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ -package biz.nynja.content.upload.token.configuration; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; - -/** - * @author Ralitsa Todorova - * - */ - -@Configuration -public class UploadTokenConfiguration { - - private final int tokenTimeToLive; - private final int fileMaxSize; - private final int maxRetries; - - @Autowired - public UploadTokenConfiguration(Environment env) { - this.tokenTimeToLive = parseProperty(env, "token.time_to_live"); - this.fileMaxSize = parseProperty(env, "token.max_file_size"); - this.maxRetries = parseProperty(env, "token.max_upload_retries"); - } - - private int parseProperty(Environment env, String property) throws InternalError { - int intProperty = 0; - try { - intProperty = Integer.parseInt(env.getRequiredProperty(property)); - } catch (NumberFormatException ex) { - throw new InternalError("Integer expected in " + property); - } - return intProperty; - - } - - public int getTokenTimeToLive() { - return tokenTimeToLive; - } - - public int getFileMaxSize() { - return fileMaxSize; - } - - public int getMaxRetries() { - return maxRetries; - } -} +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload.token.configuration; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +/** + * @author Ralitsa Todorova + * + */ + +@Configuration +public class UploadTokenConfiguration { + + private final int tokenTimeToLive; + private final int fileMaxSize; + private final int maxRetries; + + @Autowired + public UploadTokenConfiguration(Environment env) { + this.tokenTimeToLive = parseProperty(env, "token.time_to_live"); + this.fileMaxSize = parseProperty(env, "token.max_file_size"); + this.maxRetries = parseProperty(env, "token.max_upload_retries"); + } + + private int parseProperty(Environment env, String property) throws InternalError { + int intProperty = 0; + try { + intProperty = Integer.parseInt(env.getRequiredProperty(property)); + } catch (NumberFormatException ex) { + throw new InternalError("Integer expected in " + property); + } + return intProperty; + + } + + public int getTokenTimeToLive() { + return tokenTimeToLive; + } + + public int getFileMaxSize() { + return fileMaxSize; + } + + public int getMaxRetries() { + return maxRetries; + } +} diff --git a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java index 4d81a0ac1f27b6b6dedd6261a59646dfa74b036c..e37e53f258e5127a246a8765fb4c04111d56b115 100644 --- a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java +++ b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java @@ -7,6 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; +import biz.nynja.content.file.storage.Provider; + /** * @author Ralitsa Todorova * @@ -16,13 +18,31 @@ public class ContentServiceConfiguration { private final String uploadUrl; private final String downloadUrl; - private final String storageProvider; + private final Provider storageProvider; + private final int jobTTL; @Autowired public ContentServiceConfiguration(Environment env) { this.uploadUrl = env.getRequiredProperty("file.upload.url"); this.downloadUrl = env.getRequiredProperty("file.download.url"); - this.storageProvider = env.getRequiredProperty("storage.provider"); + this.storageProvider = getProvidersProperty("storage.provider", env); + this.jobTTL = parseProperty(env, "file.upload.job.ttl"); + } + + private Provider getProvidersProperty(String key, Environment env) { + String provider = env.getRequiredProperty(key); + return Provider.valueOf(provider); + } + + private int parseProperty(Environment env, String property) throws InternalError { + int intProperty = 0; + try { + intProperty = Integer.parseInt(env.getRequiredProperty(property)); + } catch (NumberFormatException ex) { + throw new InternalError("Integer expected in " + property); + } + return intProperty; + } public String getUploadUrl() { @@ -33,7 +53,11 @@ public class ContentServiceConfiguration { return downloadUrl; } - public String getStorageProvider() { + public Provider getStorageProvider() { return storageProvider; } + + public int getJobTTL() { + return jobTTL; + } } diff --git a/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java b/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java index eef546e3b11c78dbfbd725458df9a590c9d014b3..0bb2dd6473cc8372bd249173bf53fc17c8631a7b 100644 --- a/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java +++ b/src/main/java/biz/nynja/content/grpc/constants/ContentServiceConstants.java @@ -14,7 +14,7 @@ import io.grpc.Metadata; */ public class ContentServiceConstants { - public static final Metadata.Key ACCESS_TOKEN_METADATA = Metadata.Key.of("accessToken", + public static final Metadata.Key ACCESS_TOKEN_METADATA = Metadata.Key.of("Authorization", ASCII_STRING_MARSHALLER); public static final Context.Key ACCOUNT_ID = Context.key("accountId"); diff --git a/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java b/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java index f67e858e942d3be130335e310071f8601b8f5438..c0f0bfb4349dba67917b819b4de95a84048e3296 100644 --- a/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java +++ b/src/main/java/biz/nynja/content/grpc/interceptors/ContentServiceInterceptor.java @@ -22,6 +22,7 @@ import io.grpc.ServerCall; import io.grpc.ServerCall.Listener; import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; +import io.grpc.Status; /** * @author Ralitsa Todorova @@ -40,15 +41,26 @@ public class ContentServiceInterceptor implements ServerInterceptor { @Override public Listener interceptCall(ServerCall call, Metadata headers, ServerCallHandler next) { - String accessToken = headers.get(ContentServiceConstants.ACCESS_TOKEN_METADATA); + String accessToken = getAccessToken(headers.get(ContentServiceConstants.ACCESS_TOKEN_METADATA)); Context ctx = Context.current(); if (!StringUtils.isEmpty(accessToken)) { ctx = Context.current().withValue(ContentServiceConstants.ACCOUNT_ID, retrieveAccountId(accessToken)); - } + } else + call.close(Status.PERMISSION_DENIED, new Metadata()); return Contexts.interceptCall(ctx, call, headers, next); } + private String getAccessToken(String authHeader) { + // Authorization header is expected to be in format -> "Authorization" : "Bearer <>" + String accessToken = null; + try { + accessToken = authHeader.split(" ")[1]; + } catch (Exception e) { + logger.error("Unexpected Authorization Header format: {}", authHeader); + } + return accessToken; + } private String retrieveAccountId(String accessToken) { DecodedJWT decodedToken = JWT.decode(accessToken); String accountIdEncoded = decodedToken.getSubject(); diff --git a/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java b/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java index 9cfeda29fc6d93fcb3043217656ec3f4807455a5..7c91613dba14287ec006b7f6242b41534bb90aa1 100644 --- a/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java +++ b/src/main/java/biz/nynja/content/grpc/interceptors/UploadInterceptor.java @@ -11,9 +11,9 @@ import org.slf4j.LoggerFactory; import org.springframework.util.StringUtils; import biz.nynja.content.grpc.constants.ContentServiceConstants; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadTokenService; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; import io.grpc.Context; import io.grpc.Contexts; import io.grpc.Metadata; diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 722efed85502fb895e87fa236bc003f4082e8a68..b301e5b057c313e08819c4fd1962c60b4e1b0aa3 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -5,6 +5,7 @@ package biz.nynja.content.grpc.services; import static biz.nynja.content.core.validation.Validators.util; +import java.io.IOException; import java.time.Instant; import java.util.Optional; import java.util.UUID; @@ -19,6 +20,7 @@ import org.springframework.util.StringUtils; import biz.nynja.content.core.validation.Validation; import biz.nynja.content.file.metadata.FileMetadataService; import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageProvider; import biz.nynja.content.file.storage.StorageProviderPool; import biz.nynja.content.grpc.ContentServiceGrpc; @@ -30,12 +32,13 @@ import biz.nynja.content.grpc.UploadTokenResponse; import biz.nynja.content.grpc.configuration.ContentServiceConfiguration; import biz.nynja.content.grpc.constants.ContentServiceConstants; import biz.nynja.content.grpc.interceptors.UploadInterceptor; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadToken; -import biz.nynja.content.upload.token.UploadTokenResponseProvider; -import biz.nynja.content.upload.token.UploadTokenService; -import biz.nynja.content.upload.token.UploadTokenValidator; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadToken; +import biz.nynja.content.file.upload.token.UploadTokenResponseProvider; +import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.token.UploadTokenValidator; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; import io.grpc.Status; import io.grpc.StatusException; import io.grpc.stub.StreamObserver; @@ -84,6 +87,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas request.getMediaType(), request.getSize(), request.getUploadedFrom(), request.getDeviceId()); String accountId = ContentServiceConstants.ACCOUNT_ID.get(); + String fileName = uploadTokenService.encodeFileName(request.getName()); if (StringUtils.isEmpty(accountId) || !util.isValidUuid(accountId)) { uploadTokenResponseProvider.prepareErrorResponse(responseObserver, Cause.INTERNAL_SERVER_ERROR, @@ -99,7 +103,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas validationResult.getErrorMessage()); return; } - UploadToken token = new UploadToken(request.getName(), request.getMediaType().name(), request.getSize(), + UploadToken token = new UploadToken(fileName, request.getMediaType().name(), request.getSize(), request.getUploadedFrom(), request.getDeviceId()); Optional tokenResponse = uploadTokenService.prepareTokenResponse(token); if (!tokenResponse.isPresent()) { @@ -108,11 +112,12 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas logger.error("Error generating upload token:{} ", token.toString()); return; } + UUID jobId = UUID.randomUUID(); UploadStore uploadInfo = new UploadStore(tokenResponse.get(), Instant.now().toEpochMilli() + uploadTokenConfiguration.getTokenTimeToLive(), - contentSreviceConfiguration.getUploadUrl(), UUID.randomUUID(), request.getDeviceId(), - UUID.fromString(accountId), request.getName(), request.getSize(), request.getMediaType().name(), - request.getUploadedFrom(), "PENDING"); + contentSreviceConfiguration.getUploadUrl() + jobId, jobId, request.getDeviceId(), + UUID.fromString(accountId), fileName, request.getSize(), request.getMediaType().name(), + request.getUploadedFrom(), UploadStatus.PENDING); if (!uploadTokenService.storeToken(uploadInfo, false)) { uploadTokenResponseProvider.prepareErrorResponse(responseObserver, Cause.INTERNAL_SERVER_ERROR, @@ -148,6 +153,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas private int receivedBytesCount; private String fileLocation; private int reachedSize; + private Provider provider = contentSreviceConfiguration.getStorageProvider(); public UploadStreamObserver(StreamObserver responseObserver) { this.responseObserver = responseObserver; @@ -168,10 +174,10 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas try { if (storageProvider == null) { storageProvider = storageProviderPool - .getStorageProviderByType(contentSreviceConfiguration.getStorageProvider()); + .getStorageProviderByType(provider); fileName = constructFileName(uploadInfo.getFileName()); fileLocation = storageProvider.initialize(fileName); - uploadInfo.setStatus("IN_PROGRESS"); + uploadInfo.setStatus(UploadStatus.IN_PROGRESS); if (!uploadTokenService.storeToken(uploadInfo, true)) { handleStreamTermination(); responseObserver.onError(new StatusException( @@ -188,9 +194,9 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas return; } receivedBytesCount += data.length; + chunkCount++; storageProvider.write(data, fileLocation, chunkCount, (uploadInfo.getFileSize() == receivedBytesCount)); reachedSize += data.length; - chunkCount++; logger.info("Chunk {} was successfully written to output stream {}.", chunkCount, fileName); } catch (Exception e) { logger.error("Error writing data for file {}: {}", fileName, e.getMessage()); @@ -213,20 +219,31 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas Status.OUT_OF_RANGE.withDescription("Uploaded file size does not meet expectations."))); return; } - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, fileLocation); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, + storageProvider.getFileUrl(fileName), storageProvider.getProviderType()); responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); - storageProvider.close(fileLocation); + try { + storageProvider.close(fileLocation); + } catch (IOException e) { + logger.error("Error closing upload job: {}", e.getMessage()); + logger.debug("Error closing upload job: {}", e.getCause()); + } uploadTokenService.deleteToken(uploadInfo.getUploadToken()); logger.info("File sucessfully uploaded."); } private void handleStreamTermination() { uploadInfo.setRetries(uploadInfo.getRetries() + 1); - uploadInfo.setStatus("FAILED"); + uploadInfo.setStatus(UploadStatus.FAILED); uploadTokenService.storeToken(uploadInfo, false); - storageProvider.close(fileLocation); + try { + storageProvider.close(fileLocation); + } catch (IOException e) { + logger.error("Error closing upload job: {}", e.getMessage()); + logger.debug("Error closing upload job: {}", e.getCause()); + } } private String constructFileName(String fileName) { diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index bf8467532cc7e9f6c21652f4e5809b5af8241caf..b1f98b2336e5da219686fb54d3e08a2e21f7c1a7 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -15,6 +15,7 @@ spring: keyspace-name: content contact-points: localhost port: 9042 + replication: 3 token: encryptdecrypt: @@ -35,15 +36,50 @@ media-types: file: upload: - url: http://localhost:${grpc.port} + url: http://localhost:${server.port}/file/upload/ + job: + ttl: 24 # measured in hours. download: url: http://localhost:${server.port}/file/download/ storage: provider: GOOGLE local: - location: src/main/resources/ + location: src/main/resources google: uri: https://storage.googleapis.com bucket: content-service-dev upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + + +# To enable colors in Eclipse: +# spring.output.ansi.enabled=ALWAYS and in eclipse +# Help -> Install New Software... and click "Add..." to add the following URL: +# http://www.mihai-nita.net/eclipse + output: + ansi: + enabled: ALWAYS + +logging: + level: + root: INFO + org: + springframework: + web: INFO + +#Metrics related configurations +management: + endpoint: + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + exposure: + include: 'prometheus, health, info, loggers' + metrics: + export: + prometheus: + enabled: true + diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 5418080d1041c15f9b1034dfa30ba741be613581..47bd9a5b707dd4234e78a97c5b9ecde8006fdf46 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -15,6 +15,7 @@ spring: keyspace-name: ${CASSANDRA_KEYSPACE:content} contact-points: ${CASSANDRA_CONTACT_POINTS:cassandra.cassandra.svc.cluster.local} port: ${CASSANDRA_PORT:9042} + replication: ${CASSANDRA_KEYSPACE_REPLICATION:3} token: encryptdecrypt: @@ -35,15 +36,49 @@ media-types: file: upload: - url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net}:${grpc.port} + url: ${FILE_UPLOAD_URL:https://content.dev-eu.nynja.net/file/upload} + job: + ttl: 24 # measured in hours. download: url: ${FILE_DOWNLOAD_URL:https://content.dev-eu.nynja.net/file/download/} storage: provider: GOOGLE local: - location: ${LOCAL_STORAGE_LOCATION:/src/main/resources/} + location: ${LOCAL_STORAGE_LOCATION:/src/main/resources} google: uri: ${GOOGLE_STORAGE_URI:https://storage.googleapis.com} bucket: ${GOOGLE_STORAGE_BUCKET:content-service-dev} upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + +# To enable colors in Eclipse: +# spring.output.ansi.enabled=ALWAYS and in eclipse +# Help -> Install New Software... and click "Add..." to add the following URL: +# http://www.mihai-nita.net/eclipse + output: + ansi: + enabled: ALWAYS + +logging: + level: + root: INFO + org: + springframework: + web: INFO + +#Metrics related configurations +management: + endpoint: + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + exposure: + include: 'prometheus, health, info, loggers' + metrics: + export: + prometheus: + enabled: true + diff --git a/src/main/resources/logback-spring.groovy b/src/main/resources/logback-spring.groovy deleted file mode 100644 index 91261d1b5bff9fa1d8b10c9f1d0ab3688d89dca7..0000000000000000000000000000000000000000 --- a/src/main/resources/logback-spring.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ - -import ch.qos.logback.classic.encoder.PatternLayoutEncoder -import ch.qos.logback.core.rolling.RollingFileAppender -import ch.qos.logback.core.rolling.TimeBasedRollingPolicy -import ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP -import ch.qos.logback.core.status.OnConsoleStatusListener -import ch.qos.logback.classic.Level - -statusListener(OnConsoleStatusListener) - -def file = "${System.getProperty('log.dir', '')}content-service-%d.%i.log" - -appender("FILE", RollingFileAppender) { - // add a status message regarding the file property - addInfo("Starting logging to $file") - append = true - encoder(PatternLayoutEncoder) { pattern = "%d{HH:mm:ss.SSS} %level %logger - %msg%n" } - rollingPolicy(TimeBasedRollingPolicy) { - fileNamePattern = "$file" - timeBasedFileNamingAndTriggeringPolicy(SizeAndTimeBasedFNATP) { maxFileSize = "10mb" }} -} - -appender("Console-Appender", ConsoleAppender) { - encoder(PatternLayoutEncoder) { pattern = "%d{HH:mm:ss.SSS} %level %logger - %msg%n" } -} - -logger("org.springframework.cloud.config.client.ConfigServicePropertySourceLocator", Level.WARN) -root(INFO, ["Console-Appender"]) -root(Level.toLevel("${System.getProperty('log.level', 'INFO')}"), ["FILE"]) \ No newline at end of file diff --git a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java index 3683322d8e3e74de3f9a7937002b192137265b39..ad3ce0c7554ba10c1182458b05f2042f2e59a133 100644 --- a/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java +++ b/src/test/java/biz/nynja/content/file/download/FileDownloadControllerTest.java @@ -27,13 +27,15 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; +import biz.nynja.content.file.FileControllersExceptionHandler; + /** * @author Angel.Botev * */ @RunWith(SpringRunner.class) @WebMvcTest(FileDownloadController.class) -@ContextConfiguration(classes = { FileDownloadController.class, FileControllerExceptionHandler.class }) +@ContextConfiguration(classes = { FileDownloadController.class, FileControllersExceptionHandler.class }) @ActiveProfiles("dev") public class FileDownloadControllerTest { diff --git a/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f28902d535b70ab23c55cd52b6defaf0cfe28875 --- /dev/null +++ b/src/test/java/biz/nynja/content/file/upload/FileUploadControllerTest.java @@ -0,0 +1,199 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload; + +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.Optional; + +import javax.annotation.Resource; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.web.context.WebApplicationContext; + +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.FileUploadController; +import biz.nynja.content.file.upload.rest.FileUploadService; +import biz.nynja.content.file.upload.rest.UploadResponse; +import biz.nynja.content.file.upload.rest.validation.ChunkUploadValidator; +import biz.nynja.content.file.upload.token.UploadTokenService; + +/** + * @author Ralitsa Todorova + * + */ +@RunWith(SpringRunner.class) +@WebMvcTest(FileUploadController.class) +@ContextConfiguration(classes = { FileUploadController.class, ChunkUploadValidator.class, Util.class }) +@ActiveProfiles("dev") +public class FileUploadControllerTest { + + @Resource + private WebApplicationContext wac; + + @MockBean + private FileUploadService fileUploadService; + + @MockBean + private UploadTokenService uploadTokenService; + + @Autowired + private MockMvc mockMvc; + + @Autowired + @Qualifier("uploadInfo") + private UploadStore uploadInfo; + + @Autowired + @Qualifier("uploadInfoInterm") + private UploadStore uploadInfoInterm; + + @Autowired + @Qualifier("uploadResponseFinalSuccess") + private UploadResponse uploadResponseFinalSuccess; + + @Autowired + @Qualifier("uploadResponseIntermediateChunkSuccess") + private UploadResponse uploadResponseIntermediateChunkSuccess; + + private final String UPLOAD_URL = "/file/upload/"; + + @Test + public void testUploadFinalChunkCreated() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + given(fileUploadService.writeDataChunk(uploadInfo, Util.DATA_CHUNK, 1, true)) + .willReturn(uploadResponseFinalSuccess); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, true)) + .content(Util.DATA_CHUNK)).andExpect(status().isCreated()); + } + + @Test + public void testUploadFinalChunkResponse() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + given(fileUploadService.writeDataChunk(uploadInfo, Util.DATA_CHUNK, 1, true)) + .willReturn(uploadResponseFinalSuccess); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, true)) + .content(Util.DATA_CHUNK)).andExpect(status().isCreated()) + .andExpect(jsonPath("$.message").value(uploadResponseFinalSuccess.getMessage())) + .andExpect(jsonPath("$.status").value(UploadStatus.COMPLETED.name())) + .andExpect(jsonPath("$.lastSuccessfulChunk").value(1)) + .andExpect(jsonPath("$.downloadLink").value(Util.DOWNLOAD_LINK)); + } + + @Test + public void testUploadIntermediateChunkOk() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfoInterm)); + given(fileUploadService.writeDataChunk(uploadInfoInterm, Util.DATA_CHUNK, 1, false)) + .willReturn(new UploadResponse("Success", UploadStatus.IN_PROGRESS, 1)); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isCreated()) + .andExpect(jsonPath("$.status").value(UploadStatus.IN_PROGRESS.name())) + .andExpect(jsonPath("$.lastSuccessfulChunk").value(1)).andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + @Test + public void testUploadIntermediateChunkReachedFileSize() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + given(fileUploadService.writeDataChunk(uploadInfo, Util.DATA_CHUNK, 1, false)) + .willReturn(new UploadResponse("Size not match.")); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + @Test + public void testUploadWrongDataChunk() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 2, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.status").value(UploadStatus.PENDING.name())) + .andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + @Test + public void testUploadEmptyBody() throws Exception { + + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false))) + .andExpect(status().isBadRequest()); + } + + @Test + public void testUploadInvalidJobId() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.VALID_UPLOAD_TOKEN)).willReturn(Optional.of(uploadInfo)); + mockMvc.perform(put(buildUploadRequest(Util.INVALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()); + } + + @Test + public void testUploadEmptyJobId() throws Exception { + + mockMvc.perform(put(buildUploadRequest("", Util.VALID_UPLOAD_TOKEN, 1, false)).content(Util.DATA_CHUNK)) + .andExpect(status().isNotFound()); + } + + @Test + public void testUploadEmptyToken() throws Exception { + + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, "", 1, false)).content(Util.DATA_CHUNK)) + .andExpect(status().isBadRequest()); + } + + @Test + public void testUploadEmptyLast() throws Exception { + + mockMvc.perform( + put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1)).content(Util.DATA_CHUNK)) + .andExpect(status().isBadRequest()); + } + + @Test + public void testUploadEmptyPartId() throws Exception { + + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()); + } + + @Test + public void testUploadInvalidUploadToken() throws Exception { + + given(uploadTokenService.getUploadInfo(Util.INVALID_UPLOAD_TOKEN)).willReturn(Optional.empty()); + mockMvc.perform(put(buildUploadRequest(Util.VALID_UPLOAD_JOBID, Util.VALID_UPLOAD_TOKEN, 1, false)) + .content(Util.DATA_CHUNK)).andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.downloadLink").isEmpty()); + } + + private String buildUploadRequest(String jobId, String token, int partId, boolean last) { + return new StringBuilder(UPLOAD_URL).append(jobId).append("?token=").append(token).append("&partId=") + .append(partId).append("&last=").append(last).toString(); + } + + private String buildUploadRequest(String jobId, String token, boolean last) { + return new StringBuilder(UPLOAD_URL).append(jobId).append("?token=").append(token).append("&last=").append(last) + .toString(); + } + + private String buildUploadRequest(String jobId, String token, int partId) { + return new StringBuilder(UPLOAD_URL).append(jobId).append("?token=").append(token).append("&partId=") + .append(partId).toString(); + } +} diff --git a/src/test/java/biz/nynja/content/file/upload/Util.java b/src/test/java/biz/nynja/content/file/upload/Util.java new file mode 100644 index 0000000000000000000000000000000000000000..47f42fb15b80f27b96eff52462affc61454a38b3 --- /dev/null +++ b/src/test/java/biz/nynja/content/file/upload/Util.java @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.upload; + +import java.util.Random; +import java.util.UUID; + +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; + +import biz.nynja.content.file.upload.models.UploadStatus; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.rest.UploadResponse; + +/** + * @author Ralitsa Todorova + * + */ +@TestConfiguration +public class Util { + + public final static String VALID_UPLOAD_TOKEN = "qwertyuiopasdfghjklzxcvbnm"; + public final static String INVALID_UPLOAD_TOKEN = "qwertyuiopasdfghjklzxcvb"; + public final static String VALID_UPLOAD_JOBID = "3318f002-5fc1-4aff-bf70-9dbf728c29ca"; + public final static String INVALID_UPLOAD_JOBID = "3318f002-5fc1-4aff-bf70-9d"; + public final static int FILE_SIZE = 12345; + public final static String FILE_NAME = "somefilename.txt"; + public final static byte[] DATA_CHUNK = generateBytes(); + public final static String DOWNLOAD_LINK = "https://content.nynja.biz/file/download/kkk"; + + @Bean + public UploadStore uploadInfo() { + UploadStore uploadInfo = new UploadStore(); + uploadInfo.setStatus(UploadStatus.PENDING); + uploadInfo.setUploadToken(VALID_UPLOAD_TOKEN); + uploadInfo.setJobId(UUID.fromString(VALID_UPLOAD_JOBID)); + uploadInfo.setFileSize(FILE_SIZE); + uploadInfo.setFileName(FILE_NAME); + uploadInfo.setLastUploadedByte(0); + return uploadInfo; + } + + @Bean + public UploadStore uploadInfoInterm() { + UploadStore uploadInfo = new UploadStore(); + uploadInfo.setStatus(UploadStatus.PENDING); + uploadInfo.setUploadToken(VALID_UPLOAD_TOKEN); + uploadInfo.setJobId(UUID.fromString(VALID_UPLOAD_JOBID)); + uploadInfo.setFileSize(FILE_SIZE * 2); + uploadInfo.setFileName(FILE_NAME); + uploadInfo.setLastUploadedByte(0); + return uploadInfo; + } + @Bean + public UploadResponse uploadResponseIntermediateChunkSuccess() { + UploadResponse uploadResponse = new UploadResponse("Upload success.", UploadStatus.IN_PROGRESS, 1, null); + return uploadResponse; + } + + @Bean + public UploadResponse uploadResponseFinalSuccess() { + UploadResponse uploadResponse = new UploadResponse("Upload success.", UploadStatus.COMPLETED, 1, DOWNLOAD_LINK); + return uploadResponse; + } + + private static byte[] generateBytes() { + byte[] b = new byte[FILE_SIZE]; + new Random().nextBytes(b); + return b; + } +} diff --git a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java index b5dd68fb711013df8a80d90b4afb0d84b497f406..5eee2dc303371ad483374d6f4d0c1b3ffa70c752 100644 --- a/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java +++ b/src/test/java/biz/nynja/content/grpc/services/ContentServiceImplTests.java @@ -26,9 +26,9 @@ import biz.nynja.content.grpc.ErrorResponse; import biz.nynja.content.grpc.UploadTokenRequest; import biz.nynja.content.grpc.UploadTokenResponse; import biz.nynja.content.grpc.services.util.ContentServiceUtil; -import biz.nynja.content.upload.models.UploadStore; -import biz.nynja.content.upload.token.UploadToken; -import biz.nynja.content.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.models.UploadStore; +import biz.nynja.content.file.upload.token.UploadToken; +import biz.nynja.content.file.upload.token.UploadTokenService; import io.grpc.Metadata; import io.grpc.stub.MetadataUtils; @@ -83,8 +83,9 @@ public class ContentServiceImplTests extends GrpcServerTestBase { ContentServiceGrpc.ContentServiceBlockingStub contentServiceBlockingStub = ContentServiceGrpc .newBlockingStub(Optional.ofNullable(channel).orElse(inProcChannel)); Metadata header = new Metadata(); - Metadata.Key key = Metadata.Key.of("accessToken", Metadata.ASCII_STRING_MARSHALLER); - header.put(key, + Metadata.Key key = Metadata.Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER); + header.put(key, "Bearer " + + "eyJraWQiOiIyMDE4MDYwOCIsInR5cCI6IkpXVCIsImFsZyI6IkVTMjU2In0.eyJzdWIiOiJPVE5sWW1Jek5UQXRPVFZoT1MwME0ySmxMV0kzTkRjdFpEVmhPREV5TW1aalpqWTIiLCJhdWQiOiJNVEl6TVRJejphcHBDbGFzczoxMjMzMzMzIiwic2NvcGUiOiJhY2Nlc3MiLCJyb2xlcyI6W10sImlzcyI6Imh0dHBzOi8vYXV0aC5ueW5qYS5iaXovIiwiZXhwIjoxNTQyMDM4NDQ2LCJpYXQiOjE1NDIwMzQ4NDZ9._NB7HrnLeLNlHMk8bvxqrLn01TX97Y0bzylsaGiA6aEFKedKo8QciLePiLPyEEFHllpGk1bSRnqdmJjGQvuX7A"); this.contentServiceBlockingStub = MetadataUtils.attachHeaders(contentServiceBlockingStub, header); return; diff --git a/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java b/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java index 07ec4cc54cde4fbf7839e29f836c5685af438442..1a6f12dd6f039d89a59c4bca00aa80a36d6f03ca 100644 --- a/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java +++ b/src/test/java/biz/nynja/content/upload/token/UploadTokenTests.java @@ -27,8 +27,10 @@ import org.springframework.test.context.junit4.SpringRunner; import biz.nynja.content.Util; import biz.nynja.content.security.EncryptDecryptConfiguration; import biz.nynja.content.security.EncryptDecryptService; -import biz.nynja.content.upload.repositories.UploadStoreRepository; -import biz.nynja.content.upload.token.configuration.UploadTokenConfiguration; +import biz.nynja.content.file.upload.repositories.UploadStoreRepository; +import biz.nynja.content.file.upload.token.UploadToken; +import biz.nynja.content.file.upload.token.UploadTokenService; +import biz.nynja.content.file.upload.token.configuration.UploadTokenConfiguration; /** * @author Ralitsa Todorova