This project has retired. For details please refer to its Attic page.
SolrWriter xref
View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.chukwa.datacollection.writer.solr;
19  
20  import java.util.List;
21  
22  import org.apache.hadoop.chukwa.Chunk;
23  import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
24  import org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter;
25  import org.apache.hadoop.chukwa.datacollection.writer.PipelineableWriter;
26  import org.apache.hadoop.chukwa.datacollection.writer.WriterException;
27  import org.apache.hadoop.chukwa.util.ExceptionUtil;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.log4j.Logger;
30  import org.apache.solr.client.solrj.impl.CloudSolrServer;
31  import org.apache.solr.common.SolrInputDocument;
32  
33  public class SolrWriter extends PipelineableWriter {
34    private static Logger log = Logger.getLogger(SolrWriter.class);
35    private static CloudSolrServer server;
36    private static String ID = "id";
37    private static String SEQ_ID = "seqId";
38    private static String DATA_TYPE = "type";
39    private static String STREAM_NAME = "stream";
40    private static String TAGS = "tags";
41    private static String SOURCE = "source";
42    private static String DATA = "data";
43  
44    public SolrWriter() throws WriterException {
45      init(ChukwaAgent.getStaticConfiguration());
46    }
47    
48    @Override
49    public void init(Configuration c) throws WriterException {
50      String serverName = c.get("solr.cloud.address");
51      if (serverName == null) {
52        throw new WriterException("Solr server address is not defined.");
53      }
54      String collection = c.get("solr.collection", "logs");
55      server = new CloudSolrServer(serverName);
56      server.setDefaultCollection(collection);
57    }
58  
59    @Override
60    public void close() throws WriterException {
61    }
62  
63    @Override
64    public CommitStatus add(List<Chunk> chunks) throws WriterException {
65      CommitStatus rv = ChukwaWriter.COMMIT_OK;
66      for(Chunk chunk : chunks) {
67        try {
68          SolrInputDocument doc = new SolrInputDocument();
69          doc.addField(ID, chunk.getSource() + "_" + chunk.getSeqID());
70          doc.addField(TAGS, chunk.getTags());
71          doc.addField(STREAM_NAME, chunk.getStreamName());
72          doc.addField(SOURCE, chunk.getSource());
73          doc.addField(SEQ_ID, chunk.getSeqID());
74          doc.addField(DATA_TYPE, chunk.getDataType());
75          doc.addField(DATA, new String(chunk.getData()));
76          server.add(doc);
77          server.commit();
78        } catch (Exception e) {
79          log.error(ExceptionUtil.getStackTrace(e));
80          throw new WriterException("Failed to store data to Solr Cloud.");
81        }
82      }
83      if (next != null) {
84        rv = next.add(chunks); //pass data through
85      }
86      return rv;
87    }
88  }