[Bedework-commit] dumprestore r205 - in trunk: resources
src/org/bedework/dumprestore/restore
src/org/bedework/dumprestore/restore/rules
svnadmin at bedework.org
svnadmin at bedework.org
Wed May 21 14:54:54 EDT 2008
Author: douglm
Date: 2008-05-21 14:54:51 -0400 (Wed, 21 May 2008)
New Revision: 205
Modified:
trunk/resources/initbedework.xml
trunk/src/org/bedework/dumprestore/restore/HibRestore.java
trunk/src/org/bedework/dumprestore/restore/IndexerMap.java
trunk/src/org/bedework/dumprestore/restore/Restore.java
trunk/src/org/bedework/dumprestore/restore/RestoreGlobals.java
trunk/src/org/bedework/dumprestore/restore/RestoreIntf.java
trunk/src/org/bedework/dumprestore/restore/rules/CalendarFieldRule.java
trunk/src/org/bedework/dumprestore/restore/rules/CalendarRule.java
trunk/src/org/bedework/dumprestore/restore/rules/EventFieldRule.java
trunk/src/org/bedework/dumprestore/restore/rules/EventRule.java
Log:
Fix bug in lucene indexing which caused failure for collections
All a noindexing option on restore
Speed up indexing on restore by batching user indexers.
Index collections on restore
Show time spent indexing
Remove cascades option in schema for calendar children. Requires we manage that collection ourselves but avoids inefficiencies.
Add some more tests and tidy up some of the api methods.
Add code to delete a user entry.
Modified: trunk/resources/initbedework.xml
===================================================================
--- trunk/resources/initbedework.xml 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/resources/initbedework.xml 2008-05-21 18:54:51 UTC (rev 205)
@@ -122,16 +122,6 @@
<quota>10000000</quota>
</user>
<user>
- <id>8</id>
- <seq>0</seq>
- <account>douglm</account>
- <category-access>WONyA WZNnA </category-access>
- <location-access>WONyA WZNnA </location-access>
- <contact-access>WONyA WZNnA </contact-access>
- <instanceOwner>false</instanceOwner>
- <quota>10000000</quota>
- </user>
- <user>
<id>9</id>
<seq>0</seq>
<account>agrp_Engineering</account>
@@ -38641,50 +38631,6 @@
<sequence>3</sequence>
</calendar>
<calendar>
- <id>85</id>
- <seq>1</seq>
- <owner-key>
- <account>douglm</account>
- <kind>0</kind>
- </owner-key>
- <public>false</public>
- <creator-key>
- <account>douglm</account>
- <kind>0</kind>
- </creator-key>
- <calendar-path>/user</calendar-path>
- <name>douglm</name>
- <path>/user/douglm</path>
- <summary>douglm</summary>
- <calendarCollection>false</calendarCollection>
- <calType>0</calType>
- <created>20070406T183022Z</created>
- <lastmod>20070406T183022Z</lastmod>
- <sequence>0</sequence>
- </calendar>
- <calendar>
- <id>86</id>
- <seq>0</seq>
- <owner-key>
- <account>douglm</account>
- <kind>0</kind>
- </owner-key>
- <public>false</public>
- <creator-key>
- <account>douglm</account>
- <kind>0</kind>
- </creator-key>
- <calendar-path>/user/douglm</calendar-path>
- <name>calendar</name>
- <path>/user/douglm/calendar</path>
- <summary>calendar</summary>
- <calendarCollection>true</calendarCollection>
- <calType>1</calType>
- <created>20070406T183023Z</created>
- <lastmod>20070406T183023Z</lastmod>
- <sequence>0</sequence>
- </calendar>
- <calendar>
<id>87</id>
<seq>3</seq>
<owner-key>
@@ -40482,7 +40428,7 @@
<view-name>All</view-name>
</view>
</views>
- <default-calendar-path>/user/caluser</default-calendar-path>
+ <default-calendar-path>/user/caluser/calendar</default-calendar-path>
<preferredView>All</preferredView>
<pageSize>10</pageSize>
<workdayStart>0</workdayStart>
@@ -40870,6 +40816,7 @@
</view-subscriptions>
</view>
</views>
+ <default-calendar-path>/user/public-user/calendar</default-calendar-path>
<preferredView>All</preferredView>
<preferredViewPeriod>weekView</preferredViewPeriod>
<pageSize>10</pageSize>
Modified: trunk/src/org/bedework/dumprestore/restore/HibRestore.java
===================================================================
--- trunk/src/org/bedework/dumprestore/restore/HibRestore.java 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/src/org/bedework/dumprestore/restore/HibRestore.java 2008-05-21 18:54:51 UTC (rev 205)
@@ -89,9 +89,6 @@
import org.bedework.calfacade.util.AccessUtilI;
import org.bedework.dumprestore.BwDbLastmod;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.Types;
import java.util.Collection;
import org.apache.log4j.Logger;
@@ -431,10 +428,16 @@
}
}
- if (ev.getPublick()) {
- globals.getPublicIndexer().indexEntity(ev);
- } else {
- globals.getUserIndexer(ev.getOwner()).indexEntity(ev);
+ if (!globals.noIndexes) {
+ long st = System.currentTimeMillis();
+
+ if (ev.getPublick()) {
+ globals.getPublicIndexer().indexEntity(ev);
+ } else {
+ globals.getUserIndexer(ev.getOwner()).indexEntity(ev);
+ }
+
+ globals.indexTime += System.currentTimeMillis() - st;
}
// hibSave(ei.getEvent());
closeHibSess();
@@ -661,56 +664,9 @@
closeSess();
}
- public int fixUserEventsCal(BwUser u, BwCalendar cal) throws Throwable {
- /*
- PreparedStatement ps = null;
-
- try {
- ps = conn.prepareStatement(
- "UPDATE events SET calendarid=? where publicf='F' and creatorid=?");
- ps.setInt(1, cal.getId());
- ps.setInt(2, u.getId());
-
- int numUpdated = ps.executeUpdate();
-
- ps = conn.prepareStatement(
- "UPDATE users SET default_calendarid=? where userid=?");
- ps.setInt(1, cal.getId());
- ps.setInt(2, u.getId());
-
- if (ps.executeUpdate() != 1) {
- throw new Exception("Failed update of user default calendar " + u);
- }
-
- return numUpdated;
- } finally {
- if (ps != null) {
- ps.close();
- }
- }*/
- return 1;
- }
-
- /* We cannot use hibernate to set the db id here as the save will cascade
- * down all the children.
- *
- * <p>We save a skeleton copy of the calendar structure using direct jdbc
- * calls then update the structure with hibernate.
+ /* (non-Javadoc)
+ * @see org.bedework.dumprestore.restore.RestoreIntf#getCalendar(java.lang.String)
*/
- public void restoreCalendars(BwCalendar o) throws Throwable {
- if (!globals.onlyUsersMap.check(o)) {
- return;
- }
-
- openSess();
-
- restoreCalendars(o, sess.connection());
-
- sess.update(o);
-
- closeSess();
- }
-
public BwCalendar getCalendar(String path) throws Throwable {
openHibSess();
@@ -722,6 +678,9 @@
return cal;
}
+ /* (non-Javadoc)
+ * @see org.bedework.dumprestore.restore.RestoreIntf#saveRootCalendar(org.bedework.calfacade.BwCalendar)
+ */
public void saveRootCalendar(BwCalendar val) throws Throwable {
if (!globals.onlyUsersMap.check(val)) {
return;
@@ -737,11 +696,8 @@
closeHibSess();
}
- /* We cannot use hibernate to set the db id here as the save will cascade
- * down all the children.
- *
- * <p>We save a skeleton copy of the calendar structure using direct jdbc
- * calls then update the structure with hibernate.
+ /* (non-Javadoc)
+ * @see org.bedework.dumprestore.restore.RestoreIntf#addCalendar(org.bedework.calfacade.BwCalendar)
*/
public void addCalendar(BwCalendar o) throws Throwable {
if (!globals.onlyUsersMap.check(o)) {
@@ -751,103 +707,41 @@
// Ensure id not set
o.setId(CalFacadeDefs.unsavedItemKey);
- openHibSess();
+// openHibSess();
BwCalendar parent = o.getCalendar();
+// BwCalendar parent = getCalendar(o.getCalendar().getPath());
+ if (parent == null) {
+ throw new Exception("Unable to fetch parent " +
+ o.getCalendar().getPath());
+ }
- parent.addChild(o);
+// parent.addChild(o);
+ o.setCalendar(parent);
+ hibSess.save(o);
+ globals.hibBatchSize++;
// hibSess.update(parent);
- closeHibSess();
- }
+ if (!globals.noIndexes) {
+ long st = System.currentTimeMillis();
- /* ====================================================================
- * Private methods
- * ==================================================================== */
-
- private void restoreCalendars(BwCalendar val, Connection conn) throws Throwable {
- if (!globals.onlyUsersMap.check(val)) {
- return;
- }
-
- restoreCalendar(val, conn);
-
- if (val.getPublick()) {
- globals.getPublicIndexer().indexEntity(val);
- } else {
- globals.getUserIndexer(val.getOwner()).indexEntity(val);
- }
-
- Collection<BwCalendar> cals = val.getChildren();
- if (cals == null) {
- return;
- }
-
- for (BwCalendar cal: cals) {
- restoreCalendars(cal, conn);
- }
- }
-
- /* The only reason for this is the need to preserve the calendar id.
- * From 3.x on this need will not exist. Just use hibernate.
- *
- * Restore a single calendar. Don't restore children
- */
- private void restoreCalendar(BwCalendar val, Connection conn) throws Throwable {
- if (!globals.onlyUsersMap.check(val)) {
- return;
- }
-
- PreparedStatement ps = null;
-
- try {
- ps = conn.prepareStatement(
- "INSERT INTO bw_calendars " +
- "(id, bwseq, creatorid, ownerid, bwaccess, " +
- "publick, calname, bwpath, summary, description," +
- " mail_list_id, calendar_collection, parent, caltype," +
- " bw_lastmod, bwsequence, bw_created) " +
- "VALUES (?,?,?,?,?," +
- "?,?,?,?,?," +
- "?,?,?,?," +
- "?,?,?)");
-
- ps.setInt(1, val.getId());
- ps.setInt(2, val.getSeq());
- ps.setInt(3, val.getCreator().getId());
- ps.setInt(4, val.getOwner().getId());
- ps.setString(5, val.getAccess());
-
- ps.setString(6, boolVal(val.getPublick()));
- ps.setString(7, val.getName());
- ps.setString(8, val.getPath());
- ps.setString(9, val.getSummary());
- ps.setString(10, val.getDescription());
-
- ps.setString(11, val.getMailListId());
- ps.setString(12, boolVal(val.getCalendarCollection()));
-
- if (val.getCalendar() == null) {
- ps.setNull(13, Types.INTEGER);
+ if (o.getPublick()) {
+ globals.getPublicIndexer().indexEntity(o);
} else {
- ps.setInt(13, val.getCalendar().getId());
+ globals.getUserIndexer(o.getOwner()).indexEntity(o);
}
- ps.setInt(14, val.getCalType());
- ps.setString(15, val.getLastmod());
- ps.setInt(16, val.getSequence());
- ps.setString(17, val.getCreated());
- ps.executeUpdate();
- } catch (Throwable t) {
- handleException(t, "Exception restoring " + val);
- } finally {
- if (ps != null) {
- ps.close();
- }
+ globals.indexTime += System.currentTimeMillis() - st;
}
+
+ closeHibSess();
}
+ /* ====================================================================
+ * Private methods
+ * ==================================================================== */
+
private void handleException(Throwable t, String msg) throws Throwable {
log.error(msg);
log.error(this, t);
@@ -983,32 +877,6 @@
}
}
- /* Start a (possibly long-running) transaction. In the web environment
- * this might do nothing. The endTransaction method should in some way
- * check version numbers to detect concurrent updates and fail with an
- * exception.
- * /
- private void beginTransaction() throws Throwable {
- sess.beginTransaction();
- }
-
- / * End a (possibly long-running) transaction. In the web environment
- * this should in some way check version numbers to detect concurrent updates
- * and fail with an exception.
- * /
- public void endTransaction() throws Throwable {
- /* Just commit * /
- sess.commit();
- }*/
-
- private String boolVal(boolean val) {
- if (val) {
- return "T";
- }
-
- return "F";
- }
-
protected Logger getLog() {
if (log == null) {
log = Logger.getLogger(this.getClass());
Modified: trunk/src/org/bedework/dumprestore/restore/IndexerMap.java
===================================================================
--- trunk/src/org/bedework/dumprestore/restore/IndexerMap.java 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/src/org/bedework/dumprestore/restore/IndexerMap.java 2008-05-21 18:54:51 UTC (rev 205)
@@ -31,13 +31,23 @@
import org.bedework.calsvc.indexing.BwIndexerFactory;
import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
/** Globals for the restore phase
*
* @author Mike Douglass douglm - rpi.edu
* @version 1.0
*/
-public class IndexerMap extends HashMap<BwUser, BwIndexer> {
+public class IndexerMap {
+ private static final int batchSize = 50;
+
+ private static class Indexer {
+ BwIndexer idx;
+ int batchCt;
+ }
+
+ private HashMap<BwUser, Indexer> map = new HashMap<BwUser, Indexer>();
private boolean debug;
private BwSystem syspars;
@@ -53,44 +63,43 @@
}
/**
- * @param val
- */
- public BwIndexer put(BwUser user, BwIndexer val) {
- if (get(user) != null) {
- throw new RuntimeException("User already in indexer table: " + user);
- }
-
- /* We could use a least recently used list of users here so we can keep the
- * size down by flushing old entries.
- */
-
- return super.put(user, val);
- }
-
- /**
* @param key BwUser
* @return BwIndexer
*/
public BwIndexer get(BwUser key) {
- BwIndexer indexer = super.get(key);
+ Indexer indexer = map.get(key);
if (indexer != null) {
- return indexer;
+ return indexer.idx;
}
/* We could build a least recently used list or users here so we can keep the
* size down.
*/
+ indexer = new Indexer();
+
try {
- indexer = BwIndexerFactory.getIndexer(false, key, true, syspars, debug);
+ indexer.idx = BwIndexerFactory.getIndexer(false, key, true, syspars, debug);
+ indexer.idx.setBatchSize(batchSize);
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException(t);
}
- super.put(key, indexer);
+ map.put(key, indexer);
- return indexer;
+ return indexer.idx;
}
+
+ /**
+ * @throws Throwable
+ */
+ public void flush() throws Throwable {
+ Set<Map.Entry<BwUser, Indexer>> entries = map.entrySet();
+
+ for (Map.Entry<BwUser, Indexer> entry: entries) {
+ entry.getValue().idx.flush();
+ }
+ }
}
Modified: trunk/src/org/bedework/dumprestore/restore/Restore.java
===================================================================
--- trunk/src/org/bedework/dumprestore/restore/Restore.java 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/src/org/bedework/dumprestore/restore/Restore.java 2008-05-21 18:54:51 UTC (rev 205)
@@ -99,6 +99,8 @@
}
globals.getPublicIndexer().flush();
globals.getPublicIndexer().endBwBatch();
+
+ globals.flushUserIndexers();
}
void doRestore() throws Throwable {
@@ -125,6 +127,8 @@
globals.config.setDebug(true);
} else if (args[i].equals("-ndebug")) {
globals.config.setDebug(false);
+ } else if (args[i].equals("-noindexes")) {
+ globals.noIndexes = true;
} else if (args[i].equals("")) {
// null arg generated by ant
} else if (args[i].equals("-noarg")) {
Modified: trunk/src/org/bedework/dumprestore/restore/RestoreGlobals.java
===================================================================
--- trunk/src/org/bedework/dumprestore/restore/RestoreGlobals.java 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/src/org/bedework/dumprestore/restore/RestoreGlobals.java 2008-05-21 18:54:51 UTC (rev 205)
@@ -87,6 +87,18 @@
*/
public boolean entityError;
+ /** True if we should skip lucene indexing
+ */
+ public boolean noIndexes;
+
+ /** Count time spent in indexer
+ */
+ public long indexTime;
+
+ /** Time spent in indexer at start of batch
+ */
+ public long indexTimeBatchStart;
+
/** Config properties from options file.
*/
public DumpRestoreConfig config;
@@ -552,6 +564,15 @@
return idx;
}
+ /**
+ * @throws Throwable
+ */
+ public void flushUserIndexers() throws Throwable {
+ if (userIndexers != null) {
+ userIndexers.flush();
+ }
+ }
+
/** Who we are pretending to be for the core classes
*/
public BwUser currentUser;
Modified: trunk/src/org/bedework/dumprestore/restore/RestoreIntf.java
===================================================================
--- trunk/src/org/bedework/dumprestore/restore/RestoreIntf.java 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/src/org/bedework/dumprestore/restore/RestoreIntf.java 2008-05-21 18:54:51 UTC (rev 205)
@@ -312,21 +312,6 @@
*/
public void restoreDbLastmod(BwDbLastmod o) throws Throwable;
- /** Update all events for given user and set cal as the default calendar.
- *
- * @param u
- * @param cal
- * @return int num updated
- * @throws Throwable
- */
- public int fixUserEventsCal(BwUser u, BwCalendar cal) throws Throwable;
-
- /** 2.3 only
- * @param val
- * @throws Throwable
- */
- public void restoreCalendars(BwCalendar val) throws Throwable;
-
/**
* @param path
* @return BwCalendar
Modified: trunk/src/org/bedework/dumprestore/restore/rules/CalendarFieldRule.java
===================================================================
--- trunk/src/org/bedework/dumprestore/restore/rules/CalendarFieldRule.java 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/src/org/bedework/dumprestore/restore/rules/CalendarFieldRule.java 2008-05-21 18:54:51 UTC (rev 205)
@@ -90,6 +90,8 @@
ent.setLastmod(stringFld());
} else if (name.equals("sequence")) {
ent.setSequence(intFld());
+
+ } else if (name.equals("calendarCollection")) { // pre 3.5
} else {
unknownTag(name);
}
Modified: trunk/src/org/bedework/dumprestore/restore/rules/CalendarRule.java
===================================================================
--- trunk/src/org/bedework/dumprestore/restore/rules/CalendarRule.java 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/src/org/bedework/dumprestore/restore/rules/CalendarRule.java 2008-05-21 18:54:51 UTC (rev 205)
@@ -76,6 +76,13 @@
globals.calendars++;
+ if ((globals.calendars % 100) == 0) {
+ info("Restore calendar # " + globals.calendars +
+ " Time in indexer last batch " +
+ (globals.indexTime - globals.indexTimeBatchStart));
+ globals.indexTimeBatchStart = globals.indexTime;
+ }
+
fixSharableEntity(entity, "Calendar");
if ((globals.fixCaltype || globals.skipSpecialCals) &&
Modified: trunk/src/org/bedework/dumprestore/restore/rules/EventFieldRule.java
===================================================================
--- trunk/src/org/bedework/dumprestore/restore/rules/EventFieldRule.java 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/src/org/bedework/dumprestore/restore/rules/EventFieldRule.java 2008-05-21 18:54:51 UTC (rev 205)
@@ -210,9 +210,17 @@
} else if (name.equals("entityType")) {
e.setEntityType(intFld());
} else if (name.equals("name")) {
- e.setName(stringFld());
+ if (ann != null) {
+ ann.setName(stringFld());
+ } else {
+ e.setName(stringFld());
+ }
} else if (name.equals("uid")) {
- e.setUid(stringFld());
+ if (ann != null) {
+ ann.setUid(stringFld());
+ } else {
+ e.setUid(stringFld());
+ }
} else if (name.equals("classification")) {
e.setClassification(stringFld());
Modified: trunk/src/org/bedework/dumprestore/restore/rules/EventRule.java
===================================================================
--- trunk/src/org/bedework/dumprestore/restore/rules/EventRule.java 2008-05-21 18:52:29 UTC (rev 204)
+++ trunk/src/org/bedework/dumprestore/restore/rules/EventRule.java 2008-05-21 18:54:51 UTC (rev 205)
@@ -57,6 +57,7 @@
import org.bedework.calfacade.BwDateTime;
import org.bedework.calfacade.BwEvent;
import org.bedework.calfacade.BwEventAnnotation;
+import org.bedework.calfacade.BwEventObj;
import org.bedework.calfacade.BwEventProxy;
import org.bedework.calfacade.CalFacadeDefs;
import org.bedework.calfacade.svc.EventInfo;
@@ -119,6 +120,11 @@
EventInfo mei = (EventInfo)top();
ann.setTarget(mei.getEvent());
ann.setMaster(mei.getEvent());
+ } else {
+ /* Make up empty event object for us to fill in */
+ BwEvent e = new BwEventObj();
+ ann.setTarget(e);
+ ann.setMaster(e);
}
} else if (override) {
error("Restore error - override but not annotation.");
@@ -154,7 +160,10 @@
globals.events++;
if ((globals.events % 100) == 0) {
- info("Restore event # " + globals.events);
+ info("Restore event # " + globals.events +
+ " Time in indexer last batch " +
+ (globals.indexTime - globals.indexTimeBatchStart));
+ globals.indexTimeBatchStart = globals.indexTime;
}
if (!override) {
More information about the Bedework-commit
mailing list