001 /**
002 * Copyright 2010-2012 The Kuali Foundation
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016 package org.kuali.maven.wagon;
017
018 import java.io.File;
019 import java.io.FileNotFoundException;
020 import java.io.IOException;
021 import java.io.InputStream;
022 import java.io.OutputStream;
023 import java.net.URI;
024 import java.util.ArrayList;
025 import java.util.Date;
026 import java.util.List;
027
028 import org.apache.commons.io.IOUtils;
029 import org.apache.commons.lang.StringUtils;
030 import org.apache.maven.wagon.ResourceDoesNotExistException;
031 import org.apache.maven.wagon.TransferFailedException;
032 import org.apache.maven.wagon.authentication.AuthenticationException;
033 import org.apache.maven.wagon.authentication.AuthenticationInfo;
034 import org.apache.maven.wagon.proxy.ProxyInfo;
035 import org.apache.maven.wagon.repository.Repository;
036 import org.apache.maven.wagon.repository.RepositoryPermissions;
037 import org.kuali.common.threads.ExecutionStatistics;
038 import org.kuali.common.threads.ThreadHandlerContext;
039 import org.kuali.common.threads.ThreadInvoker;
040 import org.kuali.common.threads.listener.PercentCompleteListener;
041 import org.slf4j.Logger;
042 import org.slf4j.LoggerFactory;
043
044 import com.amazonaws.AmazonClientException;
045 import com.amazonaws.AmazonServiceException;
046 import com.amazonaws.auth.AWSCredentials;
047 import com.amazonaws.auth.BasicAWSCredentials;
048 import com.amazonaws.services.s3.AmazonS3Client;
049 import com.amazonaws.services.s3.internal.Mimetypes;
050 import com.amazonaws.services.s3.internal.RepeatableFileInputStream;
051 import com.amazonaws.services.s3.model.CannedAccessControlList;
052 import com.amazonaws.services.s3.model.ListObjectsRequest;
053 import com.amazonaws.services.s3.model.ObjectListing;
054 import com.amazonaws.services.s3.model.ObjectMetadata;
055 import com.amazonaws.services.s3.model.PutObjectRequest;
056 import com.amazonaws.services.s3.model.S3Object;
057 import com.amazonaws.services.s3.model.S3ObjectSummary;
058 import com.amazonaws.services.s3.transfer.TransferManager;
059
060 /**
061 * <p>
062 * An implementation of the Maven Wagon interface that is integrated with the Amazon S3 service.
063 * </p>
064 *
065 * <p>
066 * URLs that reference the S3 service should be in the form of <code>s3://bucket.name</code>. As an example
067 * <code>s3://maven.kuali.org</code> puts files into the <code>maven.kuali.org</code> bucket on the S3 service.
068 * </p>
069 *
070 * <p>
071 * This implementation uses the <code>username</code> and <code>password</code> portions of the server authentication metadata for
072 * credentials.
073 * </p>
074 *
075 * @plexus.component role="org.apache.maven.wagon.Wagon" role-hint="http" instantiation-strategy="per-lookup"
076 *
077 * @author Ben Hale
078 * @author Jeff Caddel
079 */
080 public class S3Wagon extends AbstractWagon implements RequestFactory {
081 public static final String MIN_THREADS_KEY = "maven.wagon.threads.min";
082 public static final String MAX_THREADS_KEY = "maven.wagon.threads.max";
083 public static final String DIVISOR_KEY = "maven.wagon.threads.divisor";
084 public static final int DEFAULT_MIN_THREAD_COUNT = 10;
085 public static final int DEFAULT_MAX_THREAD_COUNT = 50;
086 public static final int DEFAULT_DIVISOR = 50;
087 public static final int DEFAULT_READ_TIMEOUT = 60 * 1000;
088 public static final CannedAccessControlList DEFAULT_ACL = CannedAccessControlList.PublicRead;
089
090 ThreadInvoker invoker = new ThreadInvoker();
091 SimpleFormatter formatter = new SimpleFormatter();
092 int minThreads = getMinThreads();
093 int maxThreads = getMaxThreads();
094 int divisor = getDivisor();
095 int readTimeout = DEFAULT_READ_TIMEOUT;
096 CannedAccessControlList acl = DEFAULT_ACL;
097 TransferManager transferManager;
098
099 private static final Logger log = LoggerFactory.getLogger(S3Wagon.class);
100
101 private AmazonS3Client client;
102
103 private String bucketName;
104
105 private String basedir;
106
107 private final Mimetypes mimeTypes = Mimetypes.getInstance();
108
109 public S3Wagon() {
110 super(true);
111 S3Listener listener = new S3Listener();
112 super.addSessionListener(listener);
113 super.addTransferListener(listener);
114 }
115
116 protected void ensureBucketExists(AmazonS3Client client, String bucketName) {
117 log.debug("Looking for bucket: " + bucketName);
118 if (client.doesBucketExist(bucketName)) {
119 log.debug("Found bucket " + bucketName + " Validating permissions");
120 validatePermissions(client, bucketName);
121 } else {
122 log.info("Creating bucket " + bucketName);
123 // If we create the bucket, we "own" it and by default have the "fullcontrol" permission
124 client.createBucket(bucketName);
125 }
126 }
127
128 /**
129 * Establish that we have enough permissions on this bucket to do what we need to do
130 */
131 protected void validatePermissions(AmazonS3Client client, String bucketName) {
132 // This establishes our ability to list objects in this bucket
133 ListObjectsRequest zeroObjectsRequest = new ListObjectsRequest(bucketName, null, null, null, 0);
134 client.listObjects(zeroObjectsRequest);
135
136 /**
137 * The current AWS Java SDK does not appear to have a simple method for discovering what set of permissions the currently
138 * authenticated user has on a bucket. The AWS dev's suggest that you attempt to perform an operation that would fail if you don't
139 * have the permission in question. You would then use the success/failure of that attempt to establish what your permissions are.
140 * This is definitely not ideal and they are working on it, but it is not ready yet.
141 */
142
143 // Do something simple and quick to verify that we have write permissions on this bucket
144 // One way to do this would be to create an object in this bucket, and then immediately delete it
145 // That seems messy, inconvenient, and lame.
146
147 }
148
149 protected CannedAccessControlList getAclFromRepository(Repository repository) {
150 RepositoryPermissions permissions = repository.getPermissions();
151 if (permissions == null) {
152 return null;
153 }
154 String filePermissions = permissions.getFileMode();
155 if (StringUtils.isBlank(filePermissions)) {
156 return null;
157 }
158 return CannedAccessControlList.valueOf(filePermissions.trim());
159 }
160
161 @Override
162 protected void connectToRepository(Repository source, AuthenticationInfo auth, ProxyInfo proxy) throws AuthenticationException {
163
164 AWSCredentials credentials = getCredentials(auth);
165 this.client = new AmazonS3Client(credentials);
166 this.transferManager = new TransferManager(credentials);
167 this.bucketName = source.getHost();
168 ensureBucketExists(client, bucketName);
169 this.basedir = getBaseDir(source);
170
171 // If they've specified <filePermissions> in settings.xml, that always wins
172 CannedAccessControlList repoAcl = getAclFromRepository(source);
173 if (repoAcl != null) {
174 log.info("File permissions: " + repoAcl.name());
175 acl = repoAcl;
176 }
177 }
178
179 @Override
180 protected boolean doesRemoteResourceExist(final String resourceName) {
181 try {
182 client.getObjectMetadata(bucketName, basedir + resourceName);
183 } catch (AmazonClientException e) {
184 return false;
185 }
186 return true;
187 }
188
189 @Override
190 protected void disconnectFromRepository() {
191 // Nothing to do for S3
192 }
193
194 /**
195 * Pull an object out of an S3 bucket and write it to a file
196 */
197 @Override
198 protected void getResource(final String resourceName, final File destination, final TransferProgress progress) throws ResourceDoesNotExistException, IOException {
199 // Obtain the object from S3
200 S3Object object = null;
201 try {
202 String key = basedir + resourceName;
203 object = client.getObject(bucketName, key);
204 } catch (Exception e) {
205 throw new ResourceDoesNotExistException("Resource " + resourceName + " does not exist in the repository", e);
206 }
207
208 //
209 InputStream in = null;
210 OutputStream out = null;
211 try {
212 in = object.getObjectContent();
213 out = new TransferProgressFileOutputStream(destination, progress);
214 byte[] buffer = new byte[1024];
215 int length;
216 while ((length = in.read(buffer)) != -1) {
217 out.write(buffer, 0, length);
218 }
219 } finally {
220 IOUtils.closeQuietly(in);
221 IOUtils.closeQuietly(out);
222 }
223 }
224
225 /**
226 * Is the S3 object newer than the timestamp passed in?
227 */
228 @Override
229 protected boolean isRemoteResourceNewer(final String resourceName, final long timestamp) {
230 ObjectMetadata metadata = client.getObjectMetadata(bucketName, basedir + resourceName);
231 return metadata.getLastModified().compareTo(new Date(timestamp)) < 0;
232 }
233
234 /**
235 * List all of the objects in a given directory
236 */
237 @Override
238 protected List<String> listDirectory(String directory) throws Exception {
239 // info("directory=" + directory);
240 if (StringUtils.isBlank(directory)) {
241 directory = "";
242 }
243 String delimiter = "/";
244 String prefix = basedir + directory;
245 if (!prefix.endsWith(delimiter)) {
246 prefix += delimiter;
247 }
248 // info("prefix=" + prefix);
249 ListObjectsRequest request = new ListObjectsRequest();
250 request.setBucketName(bucketName);
251 request.setPrefix(prefix);
252 request.setDelimiter(delimiter);
253 ObjectListing objectListing = client.listObjects(request);
254 // info("truncated=" + objectListing.isTruncated());
255 // info("prefix=" + prefix);
256 // info("basedir=" + basedir);
257 List<String> fileNames = new ArrayList<String>();
258 for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
259 // info("summary.getKey()=" + summary.getKey());
260 String key = summary.getKey();
261 String relativeKey = key.startsWith(basedir) ? key.substring(basedir.length()) : key;
262 boolean add = !StringUtils.isBlank(relativeKey) && !relativeKey.equals(directory);
263 if (add) {
264 // info("Adding key - " + relativeKey);
265 fileNames.add(relativeKey);
266 }
267 }
268 for (String commonPrefix : objectListing.getCommonPrefixes()) {
269 String value = commonPrefix.startsWith(basedir) ? commonPrefix.substring(basedir.length()) : commonPrefix;
270 // info("commonPrefix=" + commonPrefix);
271 // info("relativeValue=" + relativeValue);
272 // info("Adding common prefix - " + value);
273 fileNames.add(value);
274 }
275 // StringBuilder sb = new StringBuilder();
276 // sb.append("\n");
277 // for (String fileName : fileNames) {
278 // sb.append(fileName + "\n");
279 // }
280 // info(sb.toString());
281 return fileNames;
282 }
283
284 protected void info(String msg) {
285 System.out.println("[INFO] " + msg);
286 }
287
288 /**
289 * Normalize the key to our S3 object<br>
290 * 1. Convert "./css/style.css" into "/css/style.css"<br>
291 * 2. Convert "/foo/bar/../../css/style.css" into "/css/style.css"
292 *
293 * @see java.net.URI.normalize()
294 */
295 protected String getNormalizedKey(final File source, final String destination) {
296 // Generate our bucket key for this file
297 String key = basedir + destination;
298 try {
299 String prefix = "http://s3.amazonaws.com/" + bucketName + "/";
300 String urlString = prefix + key;
301 URI rawURI = new URI(urlString);
302 URI normalizedURI = rawURI.normalize();
303 String normalized = normalizedURI.toString();
304 int pos = normalized.indexOf(prefix) + prefix.length();
305 String normalizedKey = normalized.substring(pos);
306 return normalizedKey;
307 } catch (Exception e) {
308 throw new RuntimeException(e);
309 }
310 }
311
312 protected ObjectMetadata getObjectMetadata(final File source, final String destination) {
313 // Set the mime type according to the extension of the destination file
314 String contentType = mimeTypes.getMimetype(destination);
315 long contentLength = source.length();
316
317 ObjectMetadata omd = new ObjectMetadata();
318 omd.setContentLength(contentLength);
319 omd.setContentType(contentType);
320 return omd;
321 }
322
323 /**
324 * Create a PutObjectRequest based on the PutContext
325 */
326 public PutObjectRequest getPutObjectRequest(PutFileContext context) {
327 File source = context.getSource();
328 String destination = context.getDestination();
329 TransferProgress progress = context.getProgress();
330 return getPutObjectRequest(source, destination, progress);
331 }
332
333 protected InputStream getInputStream(File source, TransferProgress progress) throws FileNotFoundException {
334 if (progress == null) {
335 return new RepeatableFileInputStream(source);
336 } else {
337 return new TransferProgressFileInputStream(source, progress);
338 }
339 }
340
341 /**
342 * Create a PutObjectRequest based on the source file and destination passed in
343 */
344 protected PutObjectRequest getPutObjectRequest(File source, String destination, TransferProgress progress) {
345 try {
346 String key = getNormalizedKey(source, destination);
347 InputStream input = getInputStream(source, progress);
348 ObjectMetadata metadata = getObjectMetadata(source, destination);
349 PutObjectRequest request = new PutObjectRequest(bucketName, key, input, metadata);
350 request.setCannedAcl(acl);
351 return request;
352 } catch (FileNotFoundException e) {
353 throw new AmazonServiceException("File not found", e);
354 }
355 }
356
357 /**
358 * On S3 there are no true "directories". An S3 bucket is essentially a Hashtable of files stored by key. The integration between a
359 * traditional file system and an S3 bucket is to use the path of the file on the local file system as the key to the file in the
360 * bucket. The S3 bucket does not contain a separate key for the directory itself.
361 */
362 public final void putDirectory(File sourceDir, String destinationDir) throws TransferFailedException {
363
364 // Examine the contents of the directory
365 List<PutFileContext> contexts = getPutFileContexts(sourceDir, destinationDir);
366 for (PutFileContext context : contexts) {
367 // Progress is tracked by the thread handler when uploading files this way
368 context.setProgress(null);
369 }
370
371 // Sum the total bytes in the directory
372 long bytes = sum(contexts);
373
374 // Show what we are up to
375 log.info(getUploadStartMsg(contexts.size(), bytes));
376
377 // Store some context for the thread handler
378 ThreadHandlerContext<PutFileContext> thc = new ThreadHandlerContext<PutFileContext>();
379 thc.setList(contexts);
380 thc.setHandler(new FileHandler());
381 thc.setMax(maxThreads);
382 thc.setMin(minThreads);
383 thc.setDivisor(divisor);
384 thc.setListener(new PercentCompleteListener<PutFileContext>());
385
386 // Invoke the threads
387 ExecutionStatistics stats = invoker.invokeThreads(thc);
388
389 // Show some stats
390 long millis = stats.getExecutionTime();
391 long count = stats.getIterationCount();
392 log.info(getUploadCompleteMsg(millis, bytes, count));
393 }
394
395 protected String getUploadCompleteMsg(long millis, long bytes, long count) {
396 String rate = formatter.getRate(millis, bytes);
397 String time = formatter.getTime(millis);
398 StringBuilder sb = new StringBuilder();
399 sb.append("Files: " + count);
400 sb.append(" Time: " + time);
401 sb.append(" Rate: " + rate);
402 return sb.toString();
403 }
404
405 protected String getUploadStartMsg(int fileCount, long bytes) {
406 StringBuilder sb = new StringBuilder();
407 sb.append("Files: " + fileCount);
408 sb.append(" Bytes: " + formatter.getSize(bytes));
409 return sb.toString();
410 }
411
412 protected int getRequestsPerThread(int threads, int requests) {
413 int requestsPerThread = requests / threads;
414 while (requestsPerThread * threads < requests) {
415 requestsPerThread++;
416 }
417 return requestsPerThread;
418 }
419
420 protected long sum(List<PutFileContext> contexts) {
421 long sum = 0;
422 for (PutFileContext context : contexts) {
423 File file = context.getSource();
424 long length = file.length();
425 sum += length;
426 }
427 return sum;
428 }
429
430 /**
431 * Store a resource into S3
432 */
433 @Override
434 protected void putResource(final File source, final String destination, final TransferProgress progress) throws IOException {
435
436 // Create a new PutObjectRequest
437 PutObjectRequest request = getPutObjectRequest(source, destination, progress);
438
439 // Upload the file to S3, using multi-part upload for large files
440 S3Utils.upload(source, request, client, transferManager);
441 }
442
443 protected String getDestinationPath(final String destination) {
444 return destination.substring(0, destination.lastIndexOf('/'));
445 }
446
447 /**
448 * Convert "/" -> ""<br>
449 * Convert "/snapshot/" -> "snapshot/"<br>
450 * Convert "/snapshot" -> "snapshot/"<br>
451 */
452 protected String getBaseDir(final Repository source) {
453 StringBuilder sb = new StringBuilder(source.getBasedir());
454 sb.deleteCharAt(0);
455 if (sb.length() == 0) {
456 return "";
457 }
458 if (sb.charAt(sb.length() - 1) != '/') {
459 sb.append('/');
460 }
461 return sb.toString();
462 }
463
464 protected String getAuthenticationErrorMessage() {
465 StringBuffer sb = new StringBuffer();
466 sb.append("The S3 wagon needs AWS Access Key set as the username and AWS Secret Key set as the password. eg:\n");
467 sb.append("<server>\n");
468 sb.append(" <id>my.server</id>\n");
469 sb.append(" <username>[AWS Access Key ID]</username>\n");
470 sb.append(" <password>[AWS Secret Access Key]</password>\n");
471 sb.append("</server>\n");
472 return sb.toString();
473 }
474
475 /**
476 * Create AWSCredentionals from the information in settings.xml
477 */
478 protected AWSCredentials getCredentials(final AuthenticationInfo authenticationInfo) throws AuthenticationException {
479 if (authenticationInfo == null) {
480 throw new AuthenticationException(getAuthenticationErrorMessage());
481 }
482 String accessKey = authenticationInfo.getUserName();
483 String secretKey = authenticationInfo.getPassword();
484 if (accessKey == null || secretKey == null) {
485 throw new AuthenticationException(getAuthenticationErrorMessage());
486 }
487 return new BasicAWSCredentials(accessKey, secretKey);
488 }
489
490 /*
491 * (non-Javadoc)
492 *
493 * @see org.kuali.maven.wagon.AbstractWagon#getPutFileContext(java.io.File, java.lang.String)
494 */
495 @Override
496 protected PutFileContext getPutFileContext(File source, String destination) {
497 PutFileContext context = super.getPutFileContext(source, destination);
498 context.setFactory(this);
499 context.setTransferManager(this.transferManager);
500 context.setClient(this.client);
501 return context;
502 }
503
504 protected int getMinThreads() {
505 return getValue(MIN_THREADS_KEY, DEFAULT_MIN_THREAD_COUNT);
506 }
507
508 protected int getMaxThreads() {
509 return getValue(MAX_THREADS_KEY, DEFAULT_MAX_THREAD_COUNT);
510 }
511
512 protected int getDivisor() {
513 return getValue(DIVISOR_KEY, DEFAULT_DIVISOR);
514 }
515
516 protected int getValue(String key, int defaultValue) {
517 String value = System.getProperty(key);
518 if (StringUtils.isEmpty(value)) {
519 return defaultValue;
520 } else {
521 return new Integer(value);
522 }
523 }
524
525 public int getReadTimeout() {
526 return readTimeout;
527 }
528
529 public void setReadTimeout(int readTimeout) {
530 this.readTimeout = readTimeout;
531 }
532
533 }