[Bedework-commit] bwcalcore r56 - in releases/bedework-3.8: .
calsvc/src/org/bedework/calsvc/indexing
svnadmin at bedework.org
svnadmin at bedework.org
Fri Apr 27 15:55:29 EDT 2012
Author: douglm
Date: 2012-04-27 15:55:28 -0400 (Fri, 27 Apr 2012)
New Revision: 56
Modified:
releases/bedework-3.8/build.xml
releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexKey.java
releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexLuceneImpl.java
releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexSolrImpl.java
releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexer.java
releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexerFactory.java
Log:
Allow enabling of solr as public events indexer.
Requires the jboss data directory to be set up correctly and an appropriately configured solr to be deployed.
Modified: releases/bedework-3.8/build.xml
===================================================================
--- releases/bedework-3.8/build.xml 2012-04-27 19:55:26 UTC (rev 55)
+++ releases/bedework-3.8/build.xml 2012-04-27 19:55:28 UTC (rev 56)
@@ -62,6 +62,9 @@
<getJar name="lucene-misc" version="2.0.0" />
<getJar name="servlet-api" version="2.4" />
+ <getJar name="solr-solrj" version="3.6.0"/>
+
+
<getJar name="bw-annotations"
version="${org.bedework.annotations.version}" project="bwannotations"
projecthome="${bedework.home}/../bwannotations" />
Modified: releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexKey.java
===================================================================
--- releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexKey.java 2012-04-27 19:55:26 UTC (rev 55)
+++ releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexKey.java 2012-04-27 19:55:28 UTC (rev 56)
@@ -6,9 +6,9 @@
Version 2.0 (the "License"); you may not use this file
except in compliance with the License. You may obtain a
copy of the License at:
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -316,7 +316,7 @@
*
* @return char value
*/
- public char getChar() {
+ private char getChar() {
if ((encoded == null) || (pos == encoded.length)) {
return (char)-1;
}
@@ -331,7 +331,7 @@
*
* @throws IndexException
*/
- public void back() throws IndexException {
+ private void back() throws IndexException {
back(1);
}
@@ -340,34 +340,34 @@
* @param n int number of chars
* @throws IndexException
*/
- public void back(final int n) throws IndexException {
- if (pos - n < 0) {
+ private void back(final int n) throws IndexException {
+ if ((pos - n) < 0) {
throw new IndexException("org.bedework.index.badKeyRewind");
}
pos -= n;
}
- /** Get current position
+ /* * Get current position
*
* @return int position
- */
+ * /
public int getPos() {
return pos;
}
- /** Set current position
+ /* * Set current position
*
* @param val int position
- */
+ * /
public void setPos(final int val) {
pos = val;
}
- /** Get number of chars remaining
+ /* * Get number of chars remaining
*
* @return int number of chars remaining
- */
+ * /
public int remaining() {
if (encoded == null) {
return 0;
@@ -375,27 +375,27 @@
return encoded.length - pos;
}
- /** Test for more
+ /* * Test for more
*
* @return boolean true for more
- */
+ * /
public boolean hasMore() {
return remaining() > 0;
}
- /** Test for no more
+ /* * Test for no more
*
* @return boolean true for no more
- */
+ * /
public boolean empty() {
return (encoded == null) || (encoded.length == 0);
}
- /** Rewind to the start
- */
- public void rewind() {
+ /* * Rewind to the start
+ * /
+ private void rewind() {
pos = 0;
- }
+ }*/
/** Return the value of a blank terminated length. On success current pos
* has been incremented.
@@ -403,7 +403,7 @@
* @return int length
* @throws IndexException
*/
- public int getLength() throws IndexException {
+ private int getLength() throws IndexException {
int res = 0;
for (;;) {
@@ -420,7 +420,7 @@
throw new IndexException("org.bedework.index.badkeychar");
}
- res = res * 10 + Character.digit(c, 10);
+ res = (res * 10) + Character.digit(c, 10);
}
return res;
@@ -431,7 +431,7 @@
* @return String decoded String value
* @throws IndexException
*/
- public String getKeyString() throws IndexException {
+ private String getKeyString() throws IndexException {
if (getChar() == 'N') {
return null;
}
@@ -448,10 +448,10 @@
return s;
}
- /** Skip a String from the encoded acl at the current position.
+ /* * Skip a String from the encoded acl at the current position.
*
* @throws IndexException
- */
+ * /
public void skipString() throws IndexException {
if (getChar() == 'N') {
return;
@@ -460,7 +460,7 @@
back();
int len = getLength();
pos += len;
- }
+ }*/
/* ====================================================================
* Encoding methods
@@ -469,7 +469,7 @@
/** Get ready to encode
*
*/
- public void startEncoding() {
+ private void startEncoding() {
caw = new CharArrayWriter();
}
@@ -478,7 +478,7 @@
* @param len
* @throws IndexException
*/
- public void encodeLength(final int len) throws IndexException {
+ private void encodeLength(final int len) throws IndexException {
try {
String slen = String.valueOf(len);
caw.write('0');
Modified: releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexLuceneImpl.java
===================================================================
--- releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexLuceneImpl.java 2012-04-27 19:55:26 UTC (rev 55)
+++ releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexLuceneImpl.java 2012-04-27 19:55:28 UTC (rev 56)
@@ -86,6 +86,7 @@
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#setBatchSize(int)
*/
+ @Override
public void setBatchSize(final int val) {
batchSize = val;
if (batchSize > 1) {
@@ -96,6 +97,7 @@
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#endBwBatch()
*/
+ @Override
public void endBwBatch() throws CalFacadeException {
try {
endBatch();
@@ -107,6 +109,7 @@
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#flush()
*/
+ @Override
public void flush() throws CalFacadeException {
if ((batch == null) || (batch.size() == 0)) {
return;
@@ -135,6 +138,7 @@
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#getKeys(int, edu.rpi.cct.misc.indexing.Index.Key[])
*/
+ @Override
public int getKeys(final int n, final Index.Key[] keys) throws CalFacadeException {
try {
return retrieve(n, keys);
@@ -146,6 +150,7 @@
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#indexEntity(java.lang.Object)
*/
+ @Override
public void indexEntity(final Object rec) throws CalFacadeException {
try {
if (batchSize > 1) {
@@ -177,6 +182,7 @@
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#unindexEntity(java.lang.Object)
*/
+ @Override
public void unindexEntity(final Object rec) throws CalFacadeException {
try {
unindexRec(rec);
@@ -192,6 +198,7 @@
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#search(java.lang.String, edu.rpi.cct.misc.indexing.SearchLimits)
*/
+ @Override
public int search(final String query, final SearchLimits limits) throws CalFacadeException {
Filter filter = null;
@@ -225,6 +232,17 @@
}
}
+ @Override
+ public void newIndex(final String name) throws CalFacadeException {
+ throw new RuntimeException("unimplemented");
+ }
+
+ @Override
+ public void swapIndex(final String index,
+ final String other) throws CalFacadeException {
+ throw new RuntimeException("unimplemented");
+ }
+
/** Called to make or fill in a Key object.
*
* @param key Possible Index.Key object for reuse
Modified: releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexSolrImpl.java
===================================================================
--- releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexSolrImpl.java 2012-04-27 19:55:26 UTC (rev 55)
+++ releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexSolrImpl.java 2012-04-27 19:55:28 UTC (rev 56)
@@ -6,9 +6,9 @@
Version 2.0 (the "License"); you may not use this file
except in compliance with the License. You may obtain a
copy of the License at:
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,20 +19,53 @@
package org.bedework.calsvc.indexing;
import org.bedework.calfacade.BwCalendar;
+import org.bedework.calfacade.BwCategory;
+import org.bedework.calfacade.BwDateTime;
import org.bedework.calfacade.BwEvent;
+import org.bedework.calfacade.BwLocation;
import org.bedework.calfacade.exc.CalFacadeException;
+import org.bedework.calfacade.svc.EventInfo;
+import org.bedework.icalendar.RecurUtil;
+import org.bedework.icalendar.RecurUtil.RecurPeriods;
import edu.rpi.cct.misc.indexing.Index;
import edu.rpi.cct.misc.indexing.IndexException;
import edu.rpi.cct.misc.indexing.SearchLimits;
+import edu.rpi.sss.util.Util;
import edu.rpi.sss.util.xml.XmlEmit;
-import org.apache.lucene.document.Document;
+import net.fortuna.ical4j.model.Period;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.protocol.HTTP;
+import org.apache.log4j.Logger;
import org.apache.lucene.index.Term;
+import org.apache.solr.client.solrj.impl.XMLResponseParser;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import javax.servlet.http.HttpServletResponse;
import javax.xml.namespace.QName;
/**
@@ -40,32 +73,78 @@
*
*/
public class BwIndexSolrImpl implements BwIndexer {
+ private transient Logger log;
+
+ private boolean debug;
+
private BwIndexKey keyConverter = new BwIndexKey();
+ private String curQuery;
+ private SearchLimits curLimits;
+
+ private StringWriter xmlWtr = null;
private int batchMaxSize = 0;
private int batchCurSize = 0;
private XmlEmit batch;
+ private Object batchLock = new Object();
+
private static final QName solrTagAdd = new QName(null, "add");
-// private static final QName solrTagDoc = new QName(null, "doc");
-// private static final QName solrTagField = new QName(null, "field");
+ private static final QName solrTagDoc = new QName(null, "doc");
+ private static final QName solrTagField = new QName(null, "field");
+ //private static final QName solrTagCommit = new QName(null, "commit");
// private static final QName solrTagOptimize = new QName(null, "optimize");
+ private String solrURL;
+
+ private int maxYears;
+ private int maxInstances;
+
+ private String targetIndex;
+ private String coreAdminPath;
+
/* Used to batch index */
/** Constructor
*
- * @param sysfilePath - Path for the index files - must exist
+ * @param solrURL - Path for the index files - must exist
* @param writeable
+ * @param maxYears
+ * @param maxInstances
+ * @param indexName - null for default
* @throws IndexException
*/
- public BwIndexSolrImpl(final String sysfilePath,
- final boolean writeable) throws IndexException {
+ public BwIndexSolrImpl(final String solrURL,
+ final boolean writeable,
+ final int maxYears,
+ final int maxInstances,
+ final String indexName,
+ final String coreAdminPath) throws IndexException {
+ debug = getLog().isDebugEnabled();
+
+ this.solrURL = solrURL;
+ this.maxYears = maxYears;
+ this.maxInstances = maxInstances;
+
+ if (indexName == null) {
+ targetIndex = "";
+ } else {
+ targetIndex = indexName + "/";
+ }
+
+ if (coreAdminPath != null) {
+ this.coreAdminPath = coreAdminPath;
+ if (coreAdminPath.endsWith("/")) {
+ // Not valid
+ this.coreAdminPath = coreAdminPath.substring(0, coreAdminPath.length() - 1);
+ }
+ }
}
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#setBatchSize(int)
*/
+ @Override
public void setBatchSize(final int val) {
batchMaxSize = val;
batchCurSize = 0;
@@ -77,104 +156,294 @@
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#endBwBatch()
*/
+ @Override
public void endBwBatch() throws CalFacadeException {
}
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#flush()
*/
+ @Override
public void flush() throws CalFacadeException {
}
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#getKeys(int, edu.rpi.cct.misc.indexing.Index.Key[])
*/
+ @Override
public int getKeys(final int n, final Index.Key[] keys) throws CalFacadeException {
- return 0;
+ SolrDocumentList sdl = search(n, keys.length);
+
+ if (sdl == null) {
+ return 0;
+ }
+
+ int num = sdl.size();
+ if (keys.length < num) {
+ // Bad result?
+ num = keys.length;
+ }
+
+ for (int i = 0; i < num; i++) {
+ SolrDocument sd = sdl.get(i);
+
+ if (sd == null) {
+ // Shouldn't happen?
+ continue;
+ }
+
+ keys[i] = makeKey(keys[i], sd);
+ }
+
+ return num;
}
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#indexEntity(java.lang.Object)
*/
+ @Override
public void indexEntity(final Object rec) throws CalFacadeException {
- /*
try {
- StringWriter xmlWtr = null;
XmlEmit xml;
if (batchMaxSize > 0) {
- synchronized(batch) {
- xml = batch;
- index(xml, rec);
+ synchronized (batchLock) {
- if (batchMaxSize == 0) {
- xml.closeTag(solrTagAdd);
+ if (batchCurSize == 0) {
+ batch = new XmlEmit();
+ xmlWtr = new StringWriter();
+ batch.startEmit(xmlWtr);
+
+ batch.openTag(solrTagAdd);
+ }
+
+ index(batch, rec);
+
+ if (batchMaxSize == batchCurSize) {
+ batch.closeTag(solrTagAdd);
+ indexAndCommit(xmlWtr.toString());
+ batchCurSize = 0;
+ }
}
- } else {
- xml = new XmlEmit();
- xmlWtr = new StringWriter();
- xml.startEmit(xmlWtr);
- index(xml, rec);
+
+ return;
}
- if (xmlWtr != null) {
- indexAndCommit(xmlWtr.toString());
- }
+ // Unbatched
+
+ xml = new XmlEmit();
+ xmlWtr = new StringWriter();
+ xml.startEmit(xmlWtr);
+
+ xml.openTag(solrTagAdd);
+
+ index(xml, rec);
+
+ xml.closeTag(solrTagAdd);
+
+ indexAndCommit(xmlWtr.toString());
} catch (IOException e) {
throw new CalFacadeException(e);
}
- */
}
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#unindexEntity(java.lang.Object)
*/
+ @Override
public void unindexEntity(final Object rec) throws CalFacadeException {
}
/* (non-Javadoc)
* @see org.bedework.calsvc.indexing.BwIndexer#search(java.lang.String, edu.rpi.cct.misc.indexing.SearchLimits)
*/
- public int search(final String query, final SearchLimits limits) throws CalFacadeException {
- return 0;
+ @Override
+ public int search(final String query,
+ final SearchLimits limits) throws CalFacadeException {
+ curQuery = query;
+ curLimits = limits;
+ SolrDocumentList sdl = search(0, 0);
+
+ if (sdl == null) {
+ return 0;
+ }
+
+ return (int)sdl.getNumFound();
}
+ /* (non-Javadoc)
+ * @see org.bedework.calsvc.indexing.BwIndexer#setCleanLocks(boolean)
+ */
+ @Override
+ public void setCleanLocks(final boolean val) {
+ throw new RuntimeException("unimplemented");
+ }
+
+ @Override
+ public void newIndex(final String name) throws CalFacadeException {
+ /* See http://wiki.apache.org/solr/CoreAdmin#CREATE
+ * for a description of this.
+ *
+ * We'll create a data directory with the same name as the new core.
+ */
+
+ String encName;
+ try {
+ encName = URLEncoder.encode(name,
+ HTTP.DEFAULT_CONTENT_CHARSET);
+ } catch (UnsupportedEncodingException Uee) {
+ throw new CalFacadeException(Uee);
+ }
+
+
+ StringBuilder sb = new StringBuilder("?action=CREATE&name=");
+ sb.append(encName);
+ sb.append("&instanceDir=bwpublic");
+ sb.append("&dataDir=");
+ sb.append(encName);
+ sb.append("/data");
+
+ InputStream str = getServer().callForStream(sb.toString(),
+ true); // coreAdmin
+
+ XMLResponseParser parser = new XMLResponseParser();
+
+ NamedList<Object> resp = parser.processResponse(new InputStreamReader(str));
+
+ SimpleOrderedMap sol = (SimpleOrderedMap)resp.get("responseHeader");
+
+ int status = (Integer)sol.get("status");
+ if (debug) {
+
+ }
+ }
+
+ @Override
+ public void swapIndex(final String index,
+ final String other) throws CalFacadeException {
+ /* See http://wiki.apache.org/solr/CoreAdmin#SWAP
+ * for a description of this.
+ */
+
+ StringBuilder sb = new StringBuilder("?action=SWAP&core=");
+ sb.append(index);
+ sb.append("&other=bwpublic");
+
+ InputStream str = getServer().callForStream(sb.toString(),
+ true); // coreAdmin
+
+ XMLResponseParser parser = new XMLResponseParser();
+
+ NamedList<Object> resp = parser.processResponse(new InputStreamReader(str));
+
+ SolrDocumentList sdl = (SolrDocumentList)resp.get("response");
+
+ if (debug) {
+
+ }
+ }
+
+ /* ========================================================================
+ * private methods
+ * ======================================================================== */
+
+ private SolrDocumentList search(final int start,
+ final int count) throws CalFacadeException {
+ StringBuilder sb = new StringBuilder();
+
+ if (curQuery != null) {
+ sb.append(curQuery);
+ }
+
+ if (curLimits != null) {
+ boolean needAnd = curQuery != null;
+
+ if (curLimits.fromDate != null) {
+ if (needAnd) {
+ sb.append(" & ");
+ needAnd = false;
+ }
+
+ sb.append(" [");
+ sb.append(toSolrDate(curLimits.fromDate));
+ }
+
+ if (curLimits.toDate != null) {
+ if (needAnd) {
+ // No start to range
+ sb.append(" & [*");
+ }
+
+ sb.append(" TO ");
+ sb.append(toSolrDate(curLimits.toDate));
+ sb.append("]");
+ } else if (curLimits.fromDate != null) {
+ // Terminate range
+
+ sb.append(" TO *]");
+ }
+ }
+
+ InputStream str = getServer().query(sb.toString(), start, count);
+
+ XMLResponseParser parser = new XMLResponseParser();
+
+ NamedList<Object> resp = parser.processResponse(new InputStreamReader(str));
+
+ return (SolrDocumentList)resp.get("response");
+ }
+
/** Called to make or fill in a Key object.
*
* @param key Possible Index.Key object for reuse
- * @param doc The retrieved Document
- * @param score The rating for this entry
+ * @param sd The retrieved document
* @return Index.Key new or reused object
- * @throws IndexException
+ * @throws CalFacadeException
*/
- public Index.Key makeKey(Index.Key key,
- final Document doc,
- final float score) throws IndexException {
+ private Index.Key makeKey(final Index.Key key,
+ final SolrDocument sd) throws CalFacadeException {
+ BwIndexKey bwkey;
+
if ((key == null) || (!(key instanceof BwIndexKey))) {
- key = new BwIndexKey();
+ bwkey = new BwIndexKey();
+ } else {
+ bwkey = (BwIndexKey)key;
}
- BwIndexKey bwkey = (BwIndexKey)key;
+ Float score = (Float)sd.getFirstValue("score");
- bwkey.setScore(score);
+ if (score != null) {
+ bwkey.setScore(score);
+ }
- String itemType = doc.get(BwIndexLuceneDefs.itemTypeInfo.getName());
- bwkey.setItemType(itemType);
+ String itemType = (String)sd.getFirstValue("itemType");
if (itemType == null) {
- throw new IndexException("org.bedework.index.noitemtype");
+ throw new CalFacadeException("org.bedework.index.noitemtype");
}
+ bwkey.setItemType(itemType);
+
+ String kval = (String)sd.getFirstValue("key");
+
+ if (kval == null) {
+ throw new CalFacadeException("org.bedework.index.noitemkey");
+ }
+
if (itemType.equals(BwIndexLuceneDefs.itemTypeCalendar)) {
- bwkey.setCalendarKey(doc.get(BwIndexLuceneDefs.keyCalendar.getName()));
+ bwkey.setCalendarKey(kval);
} else if (itemType.equals(BwIndexLuceneDefs.itemTypeEvent)) {
- bwkey.setEventKey(doc.get(BwIndexLuceneDefs.keyEvent.getName()));
+ try {
+ bwkey.setEventKey(kval);
+ } catch (IndexException ie) {
+ throw new CalFacadeException(ie);
+ }
} else {
- throw new IndexException(IndexException.unknownRecordType,
+ throw new CalFacadeException(IndexException.unknownRecordType,
itemType);
}
- return key;
+ return bwkey;
}
/** Called to make a key term for a record.
@@ -183,7 +452,7 @@
* @return Term Lucene term which uniquely identifies the record
* @throws IndexException
*/
- public Term makeKeyTerm(final Object rec) throws IndexException {
+ private Term makeKeyTerm(final Object rec) throws IndexException {
String name = makeKeyName(rec);
String key = makeKeyVal(rec);
@@ -196,7 +465,7 @@
* @return String String which uniquely identifies the record
* @throws IndexException
*/
- public String makeKeyVal(final Object rec) throws IndexException {
+ private String makeKeyVal(final Object rec) throws IndexException {
if (rec instanceof BwCalendar) {
return ((BwCalendar)rec).getPath();
}
@@ -221,7 +490,7 @@
* @return String Name for the field/term
* @throws IndexException
*/
- public String makeKeyName(final Object rec) throws IndexException {
+ private String makeKeyName(final Object rec) throws IndexException {
if (rec instanceof BwCalendar) {
return BwIndexLuceneDefs.keyCalendar.getName();
}
@@ -234,12 +503,6 @@
rec.getClass().getName());
}
- /* (non-Javadoc)
- * @see org.bedework.calsvc.indexing.BwIndexer#setCleanLocks(boolean)
- */
- public void setCleanLocks(final boolean val) {
- }
-
private synchronized void batchedIndex(final Object rec) throws CalFacadeException {
try {
if (batchCurSize == 0) {
@@ -253,18 +516,553 @@
private void index(final XmlEmit xml, final Object rec) throws CalFacadeException {
try {
- if (batchCurSize == 0) {
- xml.openTag(solrTagAdd);
+ String colPath = null;
+ Collection <BwCategory> cats = null;
+
+ String name = null;
+ String created = null;
+ String creator = null;
+ String description = null;
+ String lastmod = null;
+ String owner = null;
+ String summary = null;
+
+ String start = null;
+ String end = null;
+
+ Collection<String> recurringStarts = null;
+ Collection<String> recurringEnds = null;
+ Collection<String> recurrenceIds = null;
+
+ xml.openTag(solrTagDoc);
+
+ if (rec instanceof BwCalendar) {
+ BwCalendar cal = (BwCalendar)rec;
+
+ makeField(xml, "key", makeKeyVal(cal));
+ makeField(xml, "itemType", BwIndexLuceneDefs.itemTypeCalendar);
+
+ name = cal.getName();
+ colPath = cal.getColPath();
+ cats = cal.getCategories();
+ created = cal.getCreated();
+ creator = cal.getCreatorHref();
+ description = cal.getDescription();
+ lastmod = cal.getLastmod().getTimestamp();
+ owner = cal.getOwnerHref();
+ summary = cal.getSummary();
+ } else if (rec instanceof EventInfo) {
+ EventInfo ei = (EventInfo)rec;
+ BwEvent ev = ei.getEvent();
+
+ name = ev.getName();
+
+ makeField(xml, "key", makeKeyVal(ev));
+ makeField(xml, "itemType", BwIndexLuceneDefs.itemTypeEvent);
+
+ /*
+ if (ev instanceof BwEventProxy) {
+ // Index with the master key
+ rec.addField(BwIndexLuceneDefs.keyEventMaster,
+ makeKeyVal(((BwEventProxy)ev).getTarget()));
+ } else {
+ rec.addField(BwIndexLuceneDefs.keyEventMaster,
+ makeKeyVal(ev));
+ }
+ */
+
+ BwLocation loc = ev.getLocation();
+ if (loc != null) {
+ String s = null;
+
+ if (loc.getAddress() != null) {
+ s = loc.getAddress().getValue();
+ }
+
+ if (loc.getSubaddress() != null) {
+ if (s == null) {
+ s = loc.getSubaddress().getValue();
+ } else {
+ s = s + " " + loc.getSubaddress().getValue();
+ }
+ }
+
+ if (s != null) {
+ makeField(xml, "location_str", s);
+ }
+ }
+
+ makeField(xml, "uid", ev.getUid());
+
+ colPath = ev.getColPath();
+ cats = ev.getCategories();
+ created = ev.getCreated();
+ creator = ev.getCreatorHref();
+ description = ev.getDescription();
+ lastmod = ev.getLastmod();
+ owner = ev.getOwnerHref();
+ summary = ev.getSummary();
+
+ if (!ev.getNoStart()) {
+ start = ev.getDtstart().getDate();
+ }
+
+ if (ev.getDtend() != null) {
+ end = ev.getDtend().getDtval();
+ }
+
+ if ((ev.getRecurrenceId() == null) && ev.testRecurring()) {
+ recurringStarts = new ArrayList<String>();
+ recurringEnds = new ArrayList<String>();
+ recurrenceIds = new ArrayList<String>();
+
+ getRecurrences(ei, recurringStarts, recurringEnds, recurrenceIds);
+ }
+ } else {
+ throw new IndexException(IndexException.unknownRecordType,
+ rec.getClass().getName());
}
- if (batchMaxSize == 0) {
- xml.closeTag(solrTagAdd);
+ /* comment */
+ /* contact */
+ /* location - lat/long */
+ /* resources */
+
+ if (colPath == null) {
+ colPath = "";
}
+
+ makeField(xml, "name", name);
+ makeField(xml, "path", colPath);
+
+ indexCategories(xml, cats);
+
+ makeField(xml, "created", toSolrDate(created));
+ makeField(xml, "last_modified", toSolrDate(lastmod));
+ makeField(xml, "creator", creator);
+ makeField(xml, "owner", owner);
+ makeField(xml, "summary", summary);
+ makeField(xml, "description", description);
+
+ makeField(xml, "start", toSolrDate(start));
+ makeField(xml, "end", toSolrDate(end));
+
+ /* XXX Overrides need to be indexed separately */
+
+ if (recurrenceIds != null) {
+ for (String dt: recurrenceIds) {
+ makeField(xml, "recurrenceid", dt);
+ }
+ }
+
+ if (recurringStarts != null) {
+ for (String dt: recurringStarts) {
+ makeField(xml, "start", toSolrDate(dt.substring(0, 8) + "T000000Z"));
+ }
+ }
+
+ if (recurringEnds != null) {
+ for (String dt: recurringEnds) {
+ makeField(xml, "end", toSolrDate(dt.substring(0, 8) + "T000000Z"));
+ }
+ }
+
+ xml.closeTag(solrTagDoc);
+
+ batchCurSize++;
+ } catch (CalFacadeException cfe) {
+ throw cfe;
+ } catch (Throwable t) {
+ throw new CalFacadeException(t);
+ }
+ }
+
+ private void getRecurrences(final EventInfo val,
+ final Collection<String> recurringStarts,
+ final Collection<String> recurringEnds,
+ final Collection<String> recurrenceIds) throws IndexException {
+
+ try {
+ BwEvent ev = val.getEvent();
+
+ RecurPeriods rp = RecurUtil.getPeriods(ev, maxYears, maxInstances);
+
+ if (rp.instances.isEmpty()) {
+ // No instances for an alleged recurring event.
+ return;
+ //throw new IndexException(CalFacadeException.noRecurrenceInstances);
+ }
+
+ String stzid = ev.getDtstart().getTzid();
+
+// ev.setLatestDate(Timezones.getUtc(rp.rangeEnd.toString(),
+// stzid));
+ int instanceCt = maxInstances;
+
+ boolean dateOnly = ev.getDtstart().getDateType();
+
+ Map<String, String> overrides = new HashMap<String, String>();
+
+ if (!Util.isEmpty(val.getOverrideProxies())) {
+ for (BwEvent ov: val.getOverrideProxies()) {
+ overrides.put(ov.getRecurrenceId(), ov.getRecurrenceId());
+ }
+ }
+
+ for (Period p: rp.instances) {
+ String dtval = p.getStart().toString();
+ if (dateOnly) {
+ dtval = dtval.substring(0, 8);
+ }
+
+ BwDateTime rstart = BwDateTime.makeBwDateTime(dateOnly, dtval, stzid);
+
+ if (overrides.get(rstart.getDate()) != null) {
+ // Overrides indexed separately - skip this instance.
+ continue;
+ }
+
+ recurrenceIds.add(rstart.getDate());
+
+ dtval = p.getEnd().toString();
+ if (dateOnly) {
+ dtval = dtval.substring(0, 8);
+ }
+
+ BwDateTime rend = BwDateTime.makeBwDateTime(dateOnly, dtval, stzid);
+
+ recurringStarts.add(rstart.getDtval());
+ recurringEnds.add(rend.getDtval());
+
+ instanceCt--;
+ if (instanceCt == 0) {
+ // That's all you're getting from me
+ break;
+ }
+ }
+ } catch (Throwable t) {
+ throw new IndexException(t);
+ }
+ }
+
+ private String toSolrDate(final String val) {
+ if (val == null) {
+ return null;
+ }
+
+ // Make into form 1995-12-31T23:59:59Z
+ // from 19951231T235959Z
+ // 0 4 6 1 3
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(val.substring(0, 4));
+ sb.append("-");
+ sb.append(val.substring(4, 6));
+ sb.append("-");
+
+ if (val.length() == 8) {
+ sb.append(val.substring(6));
+ sb.append("T00:00:00.00Z");
+ } else {
+ sb.append(val.substring(6, 11));
+ sb.append(":");
+ sb.append(val.substring(11, 13));
+ sb.append(":");
+ sb.append(val.substring(13, 15));
+ sb.append(".000Z");
+ }
+
+ return sb.toString();
+ }
+
+ private void makeField(final XmlEmit xml,
+ final String name,
+ final String val) throws CalFacadeException {
+ if (val == null) {
+ return;
+ }
+
+ try {
+ xml.openTagNoNewline(solrTagField, "name", name);
+ xml.value(val);
+ xml.closeTagNoblanks(solrTagField);
} catch (IOException e) {
throw new CalFacadeException(e);
}
}
- private void indexAndCommit(final String indexInfo) throws CalFacadeException {
+ private void indexCategories(final XmlEmit xml,
+ final Collection <BwCategory> cats) throws CalFacadeException {
+ if (cats == null) {
+ return;
+ }
+
+ for (BwCategory cat: cats) {
+ makeField(xml, "category", cat.getWord().getValue());
+ }
}
+
+ SolrServer server;
+
+ private int indexAndCommit(final String indexInfo) throws CalFacadeException {
+ return getServer().postUpdate(indexInfo);
+ }
+
+ private SolrServer getServer() {
+ if (server == null) {
+ server = new SolrServer(solrURL, targetIndex, coreAdminPath);
+ }
+
+ return server;
+ }
+
+ /** CLass to allow us to call the server
+ */
+ private static class SolrServer {
+ private String serverUri;
+ private String targetIndex;
+ private String coreAdminPath;
+
+ //private static JAXBContext jc;
+
+ private HttpPost poster;
+ private HttpGet getter;
+ int status;
+ HttpResponse response;
+
+ SolrServer(final String uri,
+ final String targetIndex,
+ final String coreAdminPath) {
+ serverUri = slashIt(uri);
+ this.targetIndex = slashIt(targetIndex);
+ this.coreAdminPath = coreAdminPath; // No slash
+ }
+
+ public int postUpdate(final String xmlUpdate) throws CalFacadeException {
+ try {
+ doPost(xmlUpdate, "update");
+
+ return response.getStatusLine().getStatusCode();
+ } catch (CalFacadeException cfe) {
+ throw cfe;
+ } catch (Throwable t) {
+ throw new CalFacadeException(t);
+ }
+ }
+
+ public InputStream query(final String query,
+ final int from,
+ final int count) throws CalFacadeException {
+ try {
+ StringBuilder sb = new StringBuilder("/select?q=");
+
+ sb.append(URLEncoder.encode(query,
+ HTTP.DEFAULT_CONTENT_CHARSET));
+
+ sb.append("&start=");
+ sb.append(from);
+
+ sb.append("&rows=");
+ sb.append(count);
+
+ sb.append("&fl=*+score");
+
+ return callForStream(sb.toString(),
+ false); // coreAdmin
+ } catch (CalFacadeException cfe) {
+ throw cfe;
+ } catch (Throwable t) {
+ throw new CalFacadeException(t);
+ }
+ }
+
+ /*
+ public TimezoneListType getList(final String changedSince) throws CalFacadeException {
+ String req = "action=list";
+
+ if (changedSince != null) {
+ req = req + "&changedsince=" + changedSince;
+ }
+
+ JAXBElement jel = getXml(req);
+
+ if (jel == null) {
+ return null;
+ }
+
+ return (TimezoneListType)jel.getValue();
+ }
+
+ public InputStream getAliases() throws TimezonesException {
+ return callForStream("aliases");
+ }
+
+ public JAXBElement getXml(final String req) throws CalFacadeException {
+ InputStream is = null;
+ try {
+ is = callForStream(req);
+
+ if ((is == null) || (status != HttpServletResponse.SC_OK)) {
+ return null;
+ }
+
+ if (jc == null) {
+ synchronized (this) {
+ if (jc == null) {
+ jc = JAXBContext.newInstance(ObjectFactory.class);
+ }
+ }
+ }
+
+ Unmarshaller u = jc.createUnmarshaller();
+
+ JAXBElement jel = (JAXBElement)u.unmarshal(is);
+ return jel;
+ } catch (CalFacadeException cfe) {
+ throw cfe;
+ } catch (Throwable t) {
+ throw new CalFacadeException(t);
+ } finally {
+ if (is != null) {
+ try {
+ is.close();
+ } catch (Throwable t) {}
+ }
+ }
+ }
+ */
+
+ public InputStream callForStream(final String req,
+ final boolean coreAdmin) throws CalFacadeException {
+ try {
+ doCall(req, coreAdmin, null);
+
+ if (status != HttpServletResponse.SC_OK) {
+ return null;
+ }
+
+ HttpEntity ent = response.getEntity();
+
+ return ent.getContent();
+ } catch (CalFacadeException cfe) {
+ throw cfe;
+ } catch (Throwable t) {
+ throw new CalFacadeException(t);
+ }
+ }
+
+ public void close() throws CalFacadeException {
+ try {
+ if (response == null) {
+ return;
+ }
+
+ HttpEntity ent = response.getEntity();
+
+ if (ent != null) {
+ InputStream is = ent.getContent();
+ is.close();
+ }
+
+ getter = null;
+ response = null;
+ } catch (Throwable t) {
+ throw new CalFacadeException(t);
+ }
+ }
+
+ private void doPost(final String body,
+ final String path) throws CalFacadeException {
+ try {
+ HttpClient client = new DefaultHttpClient();
+
+ poster = new HttpPost(getUrl(false, path));
+
+ poster.setEntity(new StringEntity(body, "application/xml", "UTF-8"));
+
+ response = client.execute(poster);
+ status = response.getStatusLine().getStatusCode();
+ } catch (CalFacadeException cfe) {
+ throw cfe;
+ } catch (UnknownHostException uhe) {
+ throw new CalFacadeException(uhe);
+ } catch (Throwable t) {
+ throw new CalFacadeException(t);
+ }
+ }
+
+ private void doCall(final String req,
+ final boolean coreAdmin,
+ final String etag) throws CalFacadeException {
+ try {
+ HttpClient client = new DefaultHttpClient();
+
+ getter = new HttpGet(getUrl(coreAdmin, req));
+
+ if (etag != null) {
+ getter.addHeader(new BasicHeader("If-None-Match", etag));
+ }
+
+ response = client.execute(getter);
+ status = response.getStatusLine().getStatusCode();
+ } catch (CalFacadeException cfe) {
+ throw cfe;
+ } catch (UnknownHostException uhe) {
+ throw new CalFacadeException(uhe);
+ } catch (Throwable t) {
+ throw new CalFacadeException(t);
+ }
+ }
+
+ private String slashIt(final String s) {
+ if (s.endsWith("/")) {
+ return s;
+ }
+
+ if (s.length() == 0) {
+ return s;
+ }
+
+ return s + "/";
+ }
+
+ private String getUrl(final boolean coreAdmin,
+ final String req) throws CalFacadeException {
+ if (serverUri == null) {
+ throw new CalFacadeException("No server URI defined");
+ }
+
+ StringBuilder sb = new StringBuilder(serverUri);
+
+ if (coreAdmin) {
+ sb.append(coreAdminPath);
+ } else {
+ sb.append(targetIndex);
+ }
+
+ sb.append(req);
+
+ return sb.toString();
+ }
+ }
+
+ protected Logger getLog() {
+ if (log == null) {
+ log = Logger.getLogger(this.getClass());
+ }
+
+ return log;
+ }
+
+ protected void info(final String msg) {
+ getLog().info(msg);
+ }
+
+ protected void warn(final String msg) {
+ getLog().warn(msg);
+ }
+
+ protected void error(final String msg) {
+ getLog().error(msg);
+ }
}
Modified: releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexer.java
===================================================================
--- releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexer.java 2012-04-27 19:55:26 UTC (rev 55)
+++ releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexer.java 2012-04-27 19:55:28 UTC (rev 56)
@@ -6,9 +6,9 @@
Version 2.0 (the "License"); you may not use this file
except in compliance with the License. You may obtain a
copy of the License at:
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -30,7 +30,6 @@
*
*/
public interface BwIndexer extends Serializable {
-
/** Called to find entries that match the search string. This string may
* be a simple sequence of keywords or some sort of query the syntax of
* which is determined by the underlying implementation.
@@ -84,9 +83,24 @@
*/
public void flush() throws CalFacadeException;
- /** Indicate if we should try to clean locks.
+ /** Indicate if we should try to clean locks. (lucene)
*
* @param val
*/
public void setCleanLocks(boolean val);
+
+ /** create a new index and start using it.
+ *
+ * @param name
+ * @throws CalFacadeException
+ */
+ public void newIndex(String name) throws CalFacadeException;
+
+ /** create a new index and start using it.
+ *
+ * @param index swap this with the other
+ * @param other
+ * @throws CalFacadeException
+ */
+ public void swapIndex(String index, String other) throws CalFacadeException;
}
Modified: releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexerFactory.java
===================================================================
--- releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexerFactory.java 2012-04-27 19:55:26 UTC (rev 55)
+++ releases/bedework-3.8/calsvc/src/org/bedework/calsvc/indexing/BwIndexerFactory.java 2012-04-27 19:55:28 UTC (rev 56)
@@ -6,9 +6,9 @@
Version 2.0 (the "License"); you may not use this file
except in compliance with the License. You may obtain a
copy of the License at:
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -21,6 +21,8 @@
import org.bedework.calfacade.BwSystem;
import org.bedework.calfacade.exc.CalFacadeException;
+import edu.rpi.cct.misc.indexing.IndexLuceneImpl;
+
import org.apache.log4j.Logger;
import java.io.File;
@@ -49,13 +51,28 @@
final String principal,
final boolean writeable,
final BwSystem syspars) throws CalFacadeException {
+ if (publick && syspars.getUseSolr()) {
+ try {
+ return new BwIndexSolrImpl(syspars.getSolrURL(), writeable,
+ syspars.getMaxYears(),
+ syspars.getMaxInstances(),
+ null, // Default index
+ null); // No admin
+ } catch (Throwable t) {
+ throw new CalFacadeException(t);
+ }
+ }
+
return getIndexer(publick, principal, writeable,
syspars,
getIndexPath(syspars.getIndexRoot(),
- BwIndexLuceneDefs.currentIndexname));
+ BwIndexLuceneDefs.currentIndexname),
+ null);
}
- /** Factory method allowing us to specify the system root
+ /** Factory method allowing us to specify the system root. This should only
+ * be called from the crawler which will be indexing into an alternative
+ * index.
*
* @param publick
* @param principal
@@ -69,10 +86,19 @@
final String principal,
final boolean writeable,
final BwSystem syspars,
- final String indexRoot) throws CalFacadeException {
+ final String indexRoot,
+ final String adminPath) throws CalFacadeException {
try {
String suffix;
+ if (publick && syspars.getUseSolr()) {
+ return new BwIndexSolrImpl(syspars.getSolrURL(), writeable,
+ syspars.getMaxYears(),
+ syspars.getMaxInstances(),
+ indexRoot,
+ adminPath);
+ }
+
if (publick) {
suffix = syspars.getPublicCalendarRoot();
} else if (principal == null) {
@@ -83,7 +109,7 @@
}
String path = getIndexPath(indexRoot, suffix);
- File f = new File(path + BwIndexLuceneImpl.getPathSuffix());
+ File f = new File(path + IndexLuceneImpl.getPathSuffix());
if (f.isFile()) {
throw new CalFacadeException(CalFacadeException.notIndexDirectory,
f.getAbsolutePath());
More information about the Bedework-commit
mailing list