Source code

001/**
002 * Copyright 2014 Tampere University of Technology, Pori Department
003 * 
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 * 
008 *   http://www.apache.org/licenses/LICENSE-2.0
009 * 
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package service.tut.pori.contentanalysis;
017
018import java.io.IOException;
019import java.io.InputStream;
020import java.util.ArrayList;
021import java.util.Collection;
022import java.util.EnumSet;
023import java.util.Iterator;
024import java.util.LinkedHashSet;
025import java.util.List;
026import java.util.Set;
027import java.util.concurrent.Callable;
028import java.util.concurrent.ExecutionException;
029import java.util.concurrent.Future;
030
031import org.apache.commons.codec.net.URLCodec;
032import org.apache.commons.lang3.ArrayUtils;
033import org.apache.commons.lang3.StringUtils;
034import org.apache.http.HttpEntity;
035import org.apache.http.client.methods.CloseableHttpResponse;
036import org.apache.http.client.methods.HttpPost;
037import org.apache.http.config.SocketConfig;
038import org.apache.http.impl.client.CloseableHttpClient;
039import org.apache.http.impl.client.HttpClientBuilder;
040import org.apache.http.impl.client.HttpClients;
041import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
042import org.apache.http.util.EntityUtils;
043import org.apache.log4j.Logger;
044
045import service.tut.pori.contentanalysis.AnalysisBackend.Capability;
046import service.tut.pori.contentanalysis.CAContentCore.ServiceType;
047import service.tut.pori.contentanalysis.PhotoParameters.AnalysisType;
048import core.tut.pori.context.ServiceInitializer;
049import core.tut.pori.http.Response;
050import core.tut.pori.http.parameters.DataGroups;
051import core.tut.pori.http.parameters.Limits;
052import core.tut.pori.users.UserIdentity;
053import core.tut.pori.utils.XMLFormatter;
054
055/**
056 * Search task used to query back-ends for results. 
057 * 
058 * This class is not asynchronous and will block for the duration of execution, also this task cannot be submitted to system schedulers.
059 * 
060 */
061public class PhotoSearchTask{
062  /** Back-end capability required for search tasks */
063  public static final Capability SEARCH_CAPABILITY = Capability.PHOTO_SEARCH;
064  private static final int CONNECTION_SOCKET_TIME_OUT = 10000;  // connection socket time out in milliseconds
065  private static final XMLFormatter FORMATTER = new XMLFormatter();
066  private static final Logger LOGGER = Logger.getLogger(PhotoSearchTask.class);
067  private static final URLCodec URLCODEC = new URLCodec(core.tut.pori.http.Definitions.ENCODING_UTF8); 
068  private EnumSet<AnalysisType> _analysisTypes = null;
069  private UserIdentity _authenticatedUser = null;
070  private DataGroups _dataGroups = null;
071  private String _guid = null;
072  private Limits _limits = null;
073  private EnumSet<ServiceType> _serviceTypeFilter = null;
074  private String _url = null;
075  private long[] _userIdFilter = null;
076  
077  /**
078   * Execute this task
079   * 
080   * @return results of the search or null if no results
081   */
082  public PhotoList execute(){
083    List<AnalysisBackend> backends = ServiceInitializer.getDAOHandler().getSQLDAO(BackendDAO.class).getBackends(SEARCH_CAPABILITY);
084    if(backends == null){
085      LOGGER.warn("No backends with capability "+SEARCH_CAPABILITY.name());
086      return null;
087    }
088    
089    try (PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); CloseableHttpClient client =  HttpClientBuilder.create().setConnectionManager(cm).setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(CONNECTION_SOCKET_TIME_OUT).build()).build()){
090      ArrayList<Searcher> searchers = new ArrayList<>(backends.size());
091      String baseParameterString = generateParameterString(); // create the default parameter string
092      for(AnalysisBackend end : backends){
093        searchers.add(new Searcher(end, baseParameterString));
094      }
095      
096      List<Future<PhotoList>> results = ServiceInitializer.getExecutorHandler().getExecutor().invokeAll(searchers);
097      List<PhotoList> photoLists = new ArrayList<>(searchers.size());
098      for(Future<PhotoList> result : results){
099        if(result.isDone() && !result.isCancelled()){
100          try {
101            PhotoList r = result.get();
102            if(!PhotoList.isEmpty(r)){
103              photoLists.add(r);
104            }
105          } catch (ExecutionException ex) {
106            LOGGER.warn(ex, ex);
107          }
108        }else{
109          LOGGER.debug("Ignoring unsuccessful result.");
110        } // for
111      }
112      if(photoLists.isEmpty()){
113        LOGGER.debug("No search results.");
114      }else{
115        PhotoList photos = combineResults(_authenticatedUser, _dataGroups, _limits, photoLists);
116        return removeTargetPhoto(photos);
117      }
118    } catch (IOException | InterruptedException ex) {
119      LOGGER.error(ex, ex);
120    }
121    return null;
122  }
123  
124  /**
125   * Combine the given lists of photos. Non-existent and duplicate photos and photos which the user cannot access will be automatically removed.
126   * 
127   * @param authenticatedUser
128   * @param dataGroups
129   * @param limits note that if there is a limit for maximum number of results, the photos for the combined list will be selected randomly from the given photo lists
130   * @param photoLists the photo lists, note that only GUIDs have any meaning, all other details will be retrieved from the database based on the given data groups.
131   * @return the given photo lists combined into a single list, removing duplicates or null if no content or null collection was passed
132   */
133  private PhotoList combineResults(UserIdentity authenticatedUser, DataGroups dataGroups, Limits limits, Collection<PhotoList> photoLists){
134    if(photoLists == null || photoLists.isEmpty()){
135      return null;
136    }
137    Set<String> guids = new LinkedHashSet<>();
138    for(PhotoList photoList : photoLists){  // collect all unique GUIDs. Note: this will prioritize the FASTER back-end, and does not guarantee that MORE APPLICABLE results appear first. This is because we have no easy (defined) way to decide which results are better than others.
139      for(Photo photo : photoList.getPhotos()){
140        guids.add(photo.getGUID());
141      }
142    }
143    if(!StringUtils.isBlank(_guid)){ // if search target is given, make sure it does not appear in the results
144      guids.remove(_guid);
145    }
146    
147    PhotoList results = ServiceInitializer.getDAOHandler().getSolrDAO(PhotoDAO.class).search(authenticatedUser, dataGroups, guids, limits, null, null, null); // the back-ends are not guaranteed to return results the user has permissions to view, so re-search everything just in case
148    if(PhotoList.isEmpty(results)){
149      LOGGER.debug("No photos found.");
150      return null;
151    }
152    
153    List<Photo> sorted = new ArrayList<>(guids.size());
154    for(String guid : guids){ // sort the results back to the original order
155      Photo photo = results.getPhoto(guid);
156      if(photo == null){
157        LOGGER.warn("Ignored photo with non-existing GUID: "+guid);
158      }else{
159        sorted.add(photo);
160      }
161    }
162    results.setPhotos(sorted);
163    return results;
164  }
165  
166  /**
167   * Removes the photo with the given search term (GUID), if GUID based search was performed.
168   * 
169   * @param photos the list from which the reference photo is to be removed.
170   * @return the list with the reference photo removed or null if nothing was left after removal
171   */
172  private PhotoList removeTargetPhoto(PhotoList photos){
173    if(PhotoList.isEmpty(photos)){
174      LOGGER.debug("Empty photo list.");
175      return null;
176    }
177    if(_guid != null){
178      LOGGER.debug("Removing target photo, if present in the search results...");
179      for(Iterator<Photo> iter = photos.getPhotos().iterator();iter.hasNext();){
180        if(iter.next().getGUID().equals(_guid)){
181          iter.remove();
182        }
183      }
184    } // if
185    
186    if(PhotoList.isEmpty(photos)){
187      LOGGER.debug("No photos were left in the list, returning null..");
188      return null;
189    }else{
190      return photos;
191    }
192  }
193  
194  /**
195   * Create new search task using the GUID as a reference, the GUID is assumed to exist, no database lookup is going to be done to confirm it
196   * 
197   * @param authenticatedUser optional authenticated user
198   * @param analysisTypes 
199   * @param dataGroups optional data groups
200   * @param guid
201   * @param limits optional limits
202   * @param serviceTypeFilter optional service type filter
203   * @param userIdFilter optional user id filter, if given only the content for the specific users will searched for
204   * @return new task
205   * @throws IllegalArgumentException on bad GUID
206   */
207  public static PhotoSearchTask getTaskByGUID(UserIdentity authenticatedUser, EnumSet<AnalysisType> analysisTypes, DataGroups dataGroups, String guid, Limits limits, EnumSet<ServiceType> serviceTypeFilter, long[] userIdFilter) throws IllegalArgumentException{
208    if(StringUtils.isBlank(guid)){
209      throw new IllegalArgumentException("GUID was null or empty.");
210    }
211    PhotoSearchTask task = new PhotoSearchTask(authenticatedUser, analysisTypes, dataGroups, limits, serviceTypeFilter, userIdFilter);
212    task._guid = guid;
213    return task;
214  }
215  
216  /**
217   * Create new search task using the URL as a search parameter. The URL is assumed to be valid, and no validation is performed.
218   * 
219   * @param authenticatedUser optional authenticated user
220   * @param analysisTypes 
221   * @param dataGroups optional data groups
222   * @param limits optional limits
223   * @param serviceTypeFilter optional service type filter
224   * @param url
225   * @param userIdFilter optional user id filter, if given only the content for the specific users will searched for
226   * @return new task
227   * @throws IllegalArgumentException on bad URL
228   */
229  public static PhotoSearchTask getTaskByUrl(UserIdentity authenticatedUser, EnumSet<AnalysisType> analysisTypes, DataGroups dataGroups, Limits limits, EnumSet<ServiceType> serviceTypeFilter, String url, long[] userIdFilter) throws IllegalArgumentException{
230    if(StringUtils.isBlank(url)){
231      throw new IllegalArgumentException("Url was null or empty.");
232    }
233    PhotoSearchTask task = new PhotoSearchTask(authenticatedUser, analysisTypes, dataGroups, limits, serviceTypeFilter, userIdFilter);
234    task._url = url;
235    return task;
236  }
237  
238  /**
239   * 
240   * @param authenticatedUser
241   * @param analysisTypes 
242   * @param dataGroups
243   * @param limits
244   * @param serviceTypeFilter
245   * @param userIdFilter
246   */
247  private PhotoSearchTask(UserIdentity authenticatedUser, EnumSet<AnalysisType> analysisTypes, DataGroups dataGroups, Limits limits, EnumSet<ServiceType> serviceTypeFilter, long[] userIdFilter){
248    if(UserIdentity.isValid(authenticatedUser)){
249      _authenticatedUser = authenticatedUser;
250      LOGGER.debug("Searching with authenticated user, id: "+authenticatedUser.getUserId());
251    }else{
252      LOGGER.debug("No authenticated user.");
253    }
254    
255    if(!DataGroups.isEmpty(dataGroups)){
256      _dataGroups = dataGroups;
257    }
258    
259    _limits = limits;
260    
261    if(serviceTypeFilter != null && !serviceTypeFilter.isEmpty()){
262      LOGGER.debug("Using service type filter...");
263      _serviceTypeFilter = serviceTypeFilter;
264    }
265    
266    if(!ArrayUtils.isEmpty(userIdFilter)){
267      LOGGER.debug("Using user id filter...");
268      _userIdFilter = userIdFilter;
269    }
270    
271    if(analysisTypes != null && !analysisTypes.isEmpty()){
272      LOGGER.debug("Analysis types specified...");
273      _analysisTypes  = analysisTypes;
274    }
275  }
276  
277  /**
278   * Helper method for creating the default parameter string: filters, limits & data groups
279   * 
280   * @return parameter string or null if none
281   */
282  private String generateParameterString(){
283    StringBuilder sb = new StringBuilder();
284    if(_userIdFilter != null){
285      sb.append(core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAMS+service.tut.pori.users.Definitions.PARAMETER_USER_ID+core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAM_VALUE_SEPARATOR);
286      core.tut.pori.utils.StringUtils.append(sb, _userIdFilter, core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAM_VALUES);
287    }
288    
289    if(_serviceTypeFilter != null){
290      sb.append(core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAMS+Definitions.PARAMETER_SERVICE_ID+core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAM_VALUE_SEPARATOR);
291      core.tut.pori.utils.StringUtils.append(sb, ServiceType.toIdArray(_serviceTypeFilter), core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAM_VALUES);
292    }
293    
294    if(_limits != null){
295      sb.append(core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAMS+Limits.PARAMETER_DEFAULT_NAME+core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAM_VALUE_SEPARATOR);
296      sb.append(_limits.toLimitString());     
297    }
298    
299    if(_analysisTypes != null){
300      sb.append(core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAMS+Definitions.PARAMETER_ANALYSIS_TYPE+core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAM_VALUE_SEPARATOR);
301      Iterator<AnalysisType> iter = _analysisTypes.iterator();
302      sb.append(iter.next().toAnalysisTypeString());
303      while(iter.hasNext()){
304        sb.append(core.tut.pori.http.Definitions.SEPARATOR_URI_QUERY_PARAM_VALUES);
305        sb.append(iter.next().toAnalysisTypeString());
306      }
307    }
308    
309    if(sb.length() < 1){
310      return null;
311    }else{
312      return sb.toString();
313    }
314  }
315
316  /**
317   * Internal class, which executes the search queries to back-ends.
318   *
319   */
320  private class Searcher implements Callable<PhotoList>{
321    private AnalysisBackend _backend = null;
322    private String _baseParameterString = null;
323    
324    /**
325     * 
326     * @param end
327     * @param baseParameterString 
328     */
329    public Searcher(AnalysisBackend end, String baseParameterString){
330      _backend = end;
331      _baseParameterString = baseParameterString;
332    }
333
334    @Override
335    public PhotoList call() throws Exception {
336      StringBuilder uri = new StringBuilder(_backend.getAnalysisUri());
337      if(_url != null){
338        uri.append(Definitions.METHOD_SEARCH_SIMILAR_BY_CONTENT+"?"+Definitions.PARAMETER_URL+"=");
339        uri.append(URLCODEC.encode(_url));
340      }else{  // guid != null
341        uri.append(Definitions.METHOD_SEARCH_SIMILAR_BY_ID+"?"+Definitions.PARAMETER_GUID+"=");
342        uri.append(_guid);
343      }
344      
345      if(_baseParameterString != null){
346        uri.append(_baseParameterString);
347      } 
348      String url = uri.toString();
349      LOGGER.debug("Calling URL: "+url+" for back-end, id: "+_backend.getBackendId());
350      try (CloseableHttpClient client = HttpClients.createDefault(); CloseableHttpResponse response = client.execute(new HttpPost(url))) {
351        HttpEntity entity = response.getEntity();
352        try (InputStream content = entity.getContent()) {
353          Response r = FORMATTER.toResponse(content, PhotoList.class);
354          if(r == null){
355            LOGGER.warn("No results returned by backend, id: "+_backend.getBackendId());
356          }else{
357            return (PhotoList) r.getResponseData();
358          }
359        } finally {
360          EntityUtils.consume(entity);
361        } // try content
362      } // try response
363      return null;
364    }
365    
366  } //  class Searcher
367}