diff --git a/pom.xml b/pom.xml index 15468fb44a45cffa458793a8fcf33b84165c25ef..bc4e4eaa4663ad23587bb87082a2eaf2cd670aef 100644 --- a/pom.xml +++ b/pom.xml @@ -124,8 +124,18 @@ - org.codehaus.groovy - groovy-all + org.springframework.boot + spring-boot-starter-actuator + + + + io.micrometer + micrometer-core + + + + io.micrometer + micrometer-registry-prometheus @@ -152,11 +162,11 @@ true - - com.google.cloud - google-cloud-storage - 1.53.0 - + + com.google.cloud + google-cloud-storage + 1.53.0 + diff --git a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java index 525af77cf441b70311f71a97e08d64751b0c14ea..fd051ff91ed1800e84395b339522677030a8d738 100644 --- a/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java +++ b/src/main/java/biz/nynja/content/db/configuration/CassandraConfig.java @@ -22,6 +22,9 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Value("${spring.data.cassandra.keyspace-name}") private String keyspace; + @Value("${spring.data.cassandra.replication}") + private int replication; + @Override protected String getKeyspaceName() { return keyspace; @@ -43,6 +46,10 @@ public class CassandraConfig extends AbstractCassandraConfiguration { return port; } + private int getReplication() { + return replication; + } + @Override public SchemaAction getSchemaAction() { return SchemaAction.CREATE_IF_NOT_EXISTS; @@ -51,7 +58,7 @@ public class CassandraConfig extends AbstractCassandraConfiguration { @Override protected List getKeyspaceCreations() { CreateKeyspaceSpecification specification = CreateKeyspaceSpecification.createKeyspace(getKeyspaceName()) - .ifNotExists().withSimpleReplication(); + .ifNotExists().withSimpleReplication(getReplication()); return Arrays.asList(specification); } diff --git a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java index 483d9fea970c025cb1db5adcf143732fb86e0d68..981dff10a9e069b8618530a0f1cb100b78fe8473 100644 --- a/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java +++ b/src/main/java/biz/nynja/content/file/metadata/FileMetadataService.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import biz.nynja.content.file.metadata.dto.FileMetadata; import biz.nynja.content.file.metadata.repositories.FileMetadataReposiory; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.upload.models.UploadStore; /** @@ -29,7 +30,7 @@ public class FileMetadataService { this.fileMetadataRepository = fileMetadataRepository; } - public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl) { + public FileMetadata storeFileMetadata(UploadStore uploadInfo, String fileName, String fileUrl, Provider storageProvider) { FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setKey(UUID.randomUUID()); fileMetadata.setFileName(fileName); @@ -39,7 +40,7 @@ public class FileMetadataService { fileMetadata.setUploadTimestamp(Instant.now().toEpochMilli()); fileMetadata.setAccountId(uploadInfo.getAccountId()); fileMetadata.setDeviceId(uploadInfo.getDeviceId()); - fileMetadata.setStorage("LOCAL"); + fileMetadata.setStorage(storageProvider.name()); fileMetadata.setFileUrl(fileUrl); fileMetadataRepository.save(fileMetadata); diff --git a/src/main/java/biz/nynja/content/file/storage/Provider.java b/src/main/java/biz/nynja/content/file/storage/Provider.java new file mode 100644 index 0000000000000000000000000000000000000000..343584592fab52b95272185f175d7c006e3966f4 --- /dev/null +++ b/src/main/java/biz/nynja/content/file/storage/Provider.java @@ -0,0 +1,12 @@ +/** + * Copyright (C) 2018 Nynja Inc. All rights reserved. + */ +package biz.nynja.content.file.storage; + +/** + * @author Angel.Botev + * + */ +public enum Provider { + GOOGLE, LOCAL +} diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java index 22b699030790c5ad49a5ff580c4bcdfdcaf3e3e3..cc4df345e8d9450ff1f430e842426dc44142dc8f 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProvider.java @@ -15,4 +15,5 @@ public interface StorageProvider { public void close(String fileLocation); + public String getFileUrl(String filename); } diff --git a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java index 2a2b310edb26f8872e3c1432e91fbf00b0bb822d..461e03457b5961757a54a75f3c2181cba59c8147 100644 --- a/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java +++ b/src/main/java/biz/nynja/content/file/storage/StorageProviderPool.java @@ -23,14 +23,14 @@ public class StorageProviderPool { private static final Logger log = LoggerFactory.getLogger(StorageProviderPool.class); - private Map storageProviders = new HashMap<>(); + private Map storageProviders = new HashMap<>(); public StorageProviderPool(GoogleStorageProvider googleStorageProvider, LocalStorageProvider localStorageProvider) { - storageProviders.put("LOCAL", localStorageProvider); - storageProviders.put("GOOGLE", googleStorageProvider); + storageProviders.put(Provider.LOCAL, localStorageProvider); + storageProviders.put(Provider.GOOGLE, googleStorageProvider); } - public StorageProvider getStorageProviderByType(String storageProviderType) { + public StorageProvider getStorageProviderByType(Provider storageProviderType) { log.debug("Get storage provider = {}", storageProviderType); StorageProvider storageProvider = storageProviders.get(storageProviderType); if (storageProvider == null) { diff --git a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java index f6771524f4e48d9781ee66ad5bec41e1a29a37bd..785df48cfef74ca7a0561b4bb03b70b1b2276cd0 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/GoogleStorageProvider.java @@ -160,4 +160,12 @@ public class GoogleStorageProvider implements StorageProvider { HttpResponse resp = req.execute(); return resp; } + + @Override + public String getFileUrl(String filename) { + StringBuilder builder = new StringBuilder(); + builder.append(storageConfiguration.getGoogleStorageURI()).append("/") + .append(storageConfiguration.getGoogleBucketName()).append("/").append(filename); + return builder.toString(); + } } diff --git a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java index b9df3778ddd20da5fc75132a2456d14963150005..71573d42777fcef65f006f5ddfdc728952a8b63d 100644 --- a/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java +++ b/src/main/java/biz/nynja/content/file/storage/impl/LocalStorageProvider.java @@ -58,4 +58,9 @@ public class LocalStorageProvider implements StorageProvider { return new StringBuilder(contentStoreLocation).append(File.separator).append(fileName).toString(); } + @Override + public String getFileUrl(String filename) { + return constructFilePath(storageConfiguration.getLocalStorageLocation(), filename); + } + } diff --git a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java index 4d81a0ac1f27b6b6dedd6261a59646dfa74b036c..e520288c39ab1d8eeb7a341c06c7d109030f75e7 100644 --- a/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java +++ b/src/main/java/biz/nynja/content/grpc/configuration/ContentServiceConfiguration.java @@ -7,6 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; +import biz.nynja.content.file.storage.Provider; + /** * @author Ralitsa Todorova * @@ -16,13 +18,18 @@ public class ContentServiceConfiguration { private final String uploadUrl; private final String downloadUrl; - private final String storageProvider; + private final Provider storageProvider; @Autowired public ContentServiceConfiguration(Environment env) { this.uploadUrl = env.getRequiredProperty("file.upload.url"); this.downloadUrl = env.getRequiredProperty("file.download.url"); - this.storageProvider = env.getRequiredProperty("storage.provider"); + this.storageProvider = getProvidersProperty("storage.provider", env); + } + + private Provider getProvidersProperty(String key, Environment env) { + String provider = env.getRequiredProperty(key); + return Provider.valueOf(provider); } public String getUploadUrl() { @@ -33,7 +40,7 @@ public class ContentServiceConfiguration { return downloadUrl; } - public String getStorageProvider() { + public Provider getStorageProvider() { return storageProvider; } } diff --git a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java index 722efed85502fb895e87fa236bc003f4082e8a68..1e4f7db0cfe5e23b3fc91a18391a8ad022bb4c00 100644 --- a/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java +++ b/src/main/java/biz/nynja/content/grpc/services/ContentServiceImpl.java @@ -19,6 +19,7 @@ import org.springframework.util.StringUtils; import biz.nynja.content.core.validation.Validation; import biz.nynja.content.file.metadata.FileMetadataService; import biz.nynja.content.file.metadata.dto.FileMetadata; +import biz.nynja.content.file.storage.Provider; import biz.nynja.content.file.storage.StorageProvider; import biz.nynja.content.file.storage.StorageProviderPool; import biz.nynja.content.grpc.ContentServiceGrpc; @@ -148,6 +149,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas private int receivedBytesCount; private String fileLocation; private int reachedSize; + private Provider provider = contentSreviceConfiguration.getStorageProvider(); public UploadStreamObserver(StreamObserver responseObserver) { this.responseObserver = responseObserver; @@ -168,7 +170,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas try { if (storageProvider == null) { storageProvider = storageProviderPool - .getStorageProviderByType(contentSreviceConfiguration.getStorageProvider()); + .getStorageProviderByType(provider); fileName = constructFileName(uploadInfo.getFileName()); fileLocation = storageProvider.initialize(fileName); uploadInfo.setStatus("IN_PROGRESS"); @@ -213,7 +215,7 @@ public class ContentServiceImpl extends ContentServiceGrpc.ContentServiceImplBas Status.OUT_OF_RANGE.withDescription("Uploaded file size does not meet expectations."))); return; } - FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, fileLocation); + FileMetadata fileMetadata = fileMetadataService.storeFileMetadata(uploadInfo, fileName, storageProvider.getFileUrl(fileName), provider); responseObserver.onNext(UploadResponse.newBuilder() .setDownloadUrl(constructDownloadUrl(fileMetadata.getKey().toString())).build()); responseObserver.onCompleted(); diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index bf8467532cc7e9f6c21652f4e5809b5af8241caf..4fc172e7fab3a14e558a4768a120e1de5e6ce584 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -15,6 +15,7 @@ spring: keyspace-name: content contact-points: localhost port: 9042 + replication: 3 token: encryptdecrypt: @@ -42,8 +43,40 @@ file: storage: provider: GOOGLE local: - location: src/main/resources/ + location: src/main/resources google: uri: https://storage.googleapis.com bucket: content-service-dev upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + +# To enable colors in Eclipse: +# spring.output.ansi.enabled=ALWAYS and in eclipse +# Help -> Install New Software... and click "Add..." to add the following URL: +# http://www.mihai-nita.net/eclipse + output: + ansi: + enabled: ALWAYS + +logging: + level: + root: INFO + org: + springframework: + web: INFO + +#Metrics related configurations +management: + endpoint: + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + exposure: + include: 'prometheus, health, info, loggers' + metrics: + export: + prometheus: + enabled: true + diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml index 5418080d1041c15f9b1034dfa30ba741be613581..63f2d692c1f3ffc2fb93a83abb6a3f8813d180e4 100644 --- a/src/main/resources/application-production.yml +++ b/src/main/resources/application-production.yml @@ -15,6 +15,7 @@ spring: keyspace-name: ${CASSANDRA_KEYSPACE:content} contact-points: ${CASSANDRA_CONTACT_POINTS:cassandra.cassandra.svc.cluster.local} port: ${CASSANDRA_PORT:9042} + replication: ${CASSANDRA_KEYSPACE_REPLICATION:3} token: encryptdecrypt: @@ -42,8 +43,40 @@ file: storage: provider: GOOGLE local: - location: ${LOCAL_STORAGE_LOCATION:/src/main/resources/} + location: ${LOCAL_STORAGE_LOCATION:/src/main/resources} google: uri: ${GOOGLE_STORAGE_URI:https://storage.googleapis.com} bucket: ${GOOGLE_STORAGE_BUCKET:content-service-dev} upload_chunk_size: 262144 # measured in bytes (B) and must be a multiple of 256K bytes (that is, 262144 bytes) + +# To enable colors in Eclipse: +# spring.output.ansi.enabled=ALWAYS and in eclipse +# Help -> Install New Software... and click "Add..." to add the following URL: +# http://www.mihai-nita.net/eclipse + output: + ansi: + enabled: ALWAYS + +logging: + level: + root: INFO + org: + springframework: + web: INFO + +#Metrics related configurations +management: + endpoint: + metrics: + enabled: true + prometheus: + enabled: true + endpoints: + web: + exposure: + include: 'prometheus, health, info, loggers' + metrics: + export: + prometheus: + enabled: true + diff --git a/src/main/resources/logback-spring.groovy b/src/main/resources/logback-spring.groovy deleted file mode 100644 index 91261d1b5bff9fa1d8b10c9f1d0ab3688d89dca7..0000000000000000000000000000000000000000 --- a/src/main/resources/logback-spring.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright (C) 2018 Nynja Inc. All rights reserved. - */ - -import ch.qos.logback.classic.encoder.PatternLayoutEncoder -import ch.qos.logback.core.rolling.RollingFileAppender -import ch.qos.logback.core.rolling.TimeBasedRollingPolicy -import ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP -import ch.qos.logback.core.status.OnConsoleStatusListener -import ch.qos.logback.classic.Level - -statusListener(OnConsoleStatusListener) - -def file = "${System.getProperty('log.dir', '')}content-service-%d.%i.log" - -appender("FILE", RollingFileAppender) { - // add a status message regarding the file property - addInfo("Starting logging to $file") - append = true - encoder(PatternLayoutEncoder) { pattern = "%d{HH:mm:ss.SSS} %level %logger - %msg%n" } - rollingPolicy(TimeBasedRollingPolicy) { - fileNamePattern = "$file" - timeBasedFileNamingAndTriggeringPolicy(SizeAndTimeBasedFNATP) { maxFileSize = "10mb" }} -} - -appender("Console-Appender", ConsoleAppender) { - encoder(PatternLayoutEncoder) { pattern = "%d{HH:mm:ss.SSS} %level %logger - %msg%n" } -} - -logger("org.springframework.cloud.config.client.ConfigServicePropertySourceLocator", Level.WARN) -root(INFO, ["Console-Appender"]) -root(Level.toLevel("${System.getProperty('log.level', 'INFO')}"), ["FILE"]) \ No newline at end of file