[Bedework-commit] dumprestore r204 - in
releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore:
. rules
svnadmin at bedework.org
svnadmin at bedework.org
Wed May 21 14:52:32 EDT 2008
Author: douglm
Date: 2008-05-21 14:52:29 -0400 (Wed, 21 May 2008)
New Revision: 204
Modified:
releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/HibRestore.java
releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/IndexerMap.java
releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/Restore.java
releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/RestoreGlobals.java
releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/RestoreIntf.java
releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/rules/CalendarRule.java
releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/rules/EventRule.java
Log:
Fix bug in lucene indexing which caused failure for collections
All a noindexing option on restore
Speed up indexing on restore by batching user indexers.
Index collections on restore
Show time spent indexing
Modified: releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/HibRestore.java
===================================================================
--- releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/HibRestore.java 2008-05-15 20:52:24 UTC (rev 203)
+++ releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/HibRestore.java 2008-05-21 18:52:29 UTC (rev 204)
@@ -89,9 +89,6 @@
import org.bedework.calfacade.util.AccessUtilI;
import org.bedework.dumprestore.BwDbLastmod;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.Types;
import java.util.Collection;
import org.apache.log4j.Logger;
@@ -435,10 +432,16 @@
}
}
- if (ev.getPublick()) {
- globals.getPublicIndexer().indexEntity(ev);
- } else {
- globals.getUserIndexer(ev.getOwner()).indexEntity(ev);
+ if (!globals.noIndexes) {
+ long st = System.currentTimeMillis();
+
+ if (ev.getPublick()) {
+ globals.getPublicIndexer().indexEntity(ev);
+ } else {
+ globals.getUserIndexer(ev.getOwner()).indexEntity(ev);
+ }
+
+ globals.indexTime += System.currentTimeMillis() - st;
}
// hibSave(ei.getEvent());
closeHibSess();
@@ -665,56 +668,6 @@
closeSess();
}
- public int fixUserEventsCal(BwUser u, BwCalendar cal) throws Throwable {
- /*
- PreparedStatement ps = null;
-
- try {
- ps = conn.prepareStatement(
- "UPDATE events SET calendarid=? where publicf='F' and creatorid=?");
- ps.setInt(1, cal.getId());
- ps.setInt(2, u.getId());
-
- int numUpdated = ps.executeUpdate();
-
- ps = conn.prepareStatement(
- "UPDATE users SET default_calendarid=? where userid=?");
- ps.setInt(1, cal.getId());
- ps.setInt(2, u.getId());
-
- if (ps.executeUpdate() != 1) {
- throw new Exception("Failed update of user default calendar " + u);
- }
-
- return numUpdated;
- } finally {
- if (ps != null) {
- ps.close();
- }
- }*/
- return 1;
- }
-
- /* We cannot use hibernate to set the db id here as the save will cascade
- * down all the children.
- *
- * <p>We save a skeleton copy of the calendar structure using direct jdbc
- * calls then update the structure with hibernate.
- */
- public void restoreCalendars(BwCalendar o) throws Throwable {
- if (!globals.onlyUsersMap.check(o)) {
- return;
- }
-
- openSess();
-
- restoreCalendars(o, sess.connection());
-
- sess.update(o);
-
- closeSess();
- }
-
public BwCalendar getCalendar(String path) throws Throwable {
openHibSess();
@@ -726,6 +679,9 @@
return cal;
}
+ /* (non-Javadoc)
+ * @see org.bedework.dumprestore.restore.RestoreIntf#saveRootCalendar(org.bedework.calfacade.BwCalendar)
+ */
public void saveRootCalendar(BwCalendar val) throws Throwable {
if (!globals.onlyUsersMap.check(val)) {
return;
@@ -741,11 +697,8 @@
closeHibSess();
}
- /* We cannot use hibernate to set the db id here as the save will cascade
- * down all the children.
- *
- * <p>We save a skeleton copy of the calendar structure using direct jdbc
- * calls then update the structure with hibernate.
+ /* (non-Javadoc)
+ * @see org.bedework.dumprestore.restore.RestoreIntf#addCalendar(org.bedework.calfacade.BwCalendar)
*/
public void addCalendar(BwCalendar o) throws Throwable {
if (!globals.onlyUsersMap.check(o)) {
@@ -763,95 +716,25 @@
// hibSess.update(parent);
- closeHibSess();
- }
+ if (!globals.noIndexes) {
+ long st = System.currentTimeMillis();
- /* ====================================================================
- * Private methods
- * ==================================================================== */
-
- private void restoreCalendars(BwCalendar val, Connection conn) throws Throwable {
- if (!globals.onlyUsersMap.check(val)) {
- return;
- }
-
- restoreCalendar(val, conn);
-
- if (val.getPublick()) {
- globals.getPublicIndexer().indexEntity(val);
- } else {
- globals.getUserIndexer(val.getOwner()).indexEntity(val);
- }
-
- Collection<BwCalendar> cals = val.getChildren();
- if (cals == null) {
- return;
- }
-
- for (BwCalendar cal: cals) {
- restoreCalendars(cal, conn);
- }
- }
-
- /* The only reason for this is the need to preserve the calendar id.
- * From 3.x on this need will not exist. Just use hibernate.
- *
- * Restore a single calendar. Don't restore children
- */
- private void restoreCalendar(BwCalendar val, Connection conn) throws Throwable {
- if (!globals.onlyUsersMap.check(val)) {
- return;
- }
-
- PreparedStatement ps = null;
-
- try {
- ps = conn.prepareStatement(
- "INSERT INTO bw_calendars " +
- "(id, bwseq, creatorid, ownerid, bwaccess, " +
- "publick, calname, bwpath, summary, description," +
- " mail_list_id, calendar_collection, parent, caltype," +
- " bw_lastmod, bwsequence, bw_created) " +
- "VALUES (?,?,?,?,?," +
- "?,?,?,?,?," +
- "?,?,?,?," +
- "?,?,?)");
-
- ps.setInt(1, val.getId());
- ps.setInt(2, val.getSeq());
- ps.setInt(3, val.getCreator().getId());
- ps.setInt(4, val.getOwner().getId());
- ps.setString(5, val.getAccess());
-
- ps.setString(6, boolVal(val.getPublick()));
- ps.setString(7, val.getName());
- ps.setString(8, val.getPath());
- ps.setString(9, val.getSummary());
- ps.setString(10, val.getDescription());
-
- ps.setString(11, val.getMailListId());
- ps.setString(12, boolVal(val.getCalendarCollection()));
-
- if (val.getCalendar() == null) {
- ps.setNull(13, Types.INTEGER);
+ if (o.getPublick()) {
+ globals.getPublicIndexer().indexEntity(o);
} else {
- ps.setInt(13, val.getCalendar().getId());
+ globals.getUserIndexer(o.getOwner()).indexEntity(o);
}
- ps.setInt(14, val.getCalType());
- ps.setString(15, val.getLastmod());
- ps.setInt(16, val.getSequence());
- ps.setString(17, val.getCreated());
- ps.executeUpdate();
- } catch (Throwable t) {
- handleException(t, "Exception restoring " + val);
- } finally {
- if (ps != null) {
- ps.close();
- }
+ globals.indexTime += System.currentTimeMillis() - st;
}
+
+ closeHibSess();
}
+ /* ====================================================================
+ * Private methods
+ * ==================================================================== */
+
private void handleException(Throwable t, String msg) throws Throwable {
log.error(msg);
log.error(this, t);
@@ -987,32 +870,6 @@
}
}
- /* Start a (possibly long-running) transaction. In the web environment
- * this might do nothing. The endTransaction method should in some way
- * check version numbers to detect concurrent updates and fail with an
- * exception.
- * /
- private void beginTransaction() throws Throwable {
- sess.beginTransaction();
- }
-
- / * End a (possibly long-running) transaction. In the web environment
- * this should in some way check version numbers to detect concurrent updates
- * and fail with an exception.
- * /
- public void endTransaction() throws Throwable {
- /* Just commit * /
- sess.commit();
- }*/
-
- private String boolVal(boolean val) {
- if (val) {
- return "T";
- }
-
- return "F";
- }
-
protected Logger getLog() {
if (log == null) {
log = Logger.getLogger(this.getClass());
Modified: releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/IndexerMap.java
===================================================================
--- releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/IndexerMap.java 2008-05-15 20:52:24 UTC (rev 203)
+++ releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/IndexerMap.java 2008-05-21 18:52:29 UTC (rev 204)
@@ -1,33 +1,5 @@
-/*
- Copyright (c) 2000-2005 University of Washington. All rights reserved.
-
- Redistribution and use of this distribution in source and binary forms,
- with or without modification, are permitted provided that:
-
- The above copyright notice and this permission notice appear in
- all copies and supporting documentation;
-
- The name, identifiers, and trademarks of the University of Washington
- are not used in advertising or publicity without the express prior
- written permission of the University of Washington;
-
- Recipients acknowledge that this distribution is made available as a
- research courtesy, "as is", potentially with defects, without
- any obligation on the part of the University of Washington to
- provide support, services, or repair;
-
- THE UNIVERSITY OF WASHINGTON DISCLAIMS ALL WARRANTIES, EXPRESS OR
- IMPLIED, WITH REGARD TO THIS SOFTWARE, INCLUDING WITHOUT LIMITATION
- ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
- PARTICULAR PURPOSE, AND IN NO EVENT SHALL THE UNIVERSITY OF
- WASHINGTON BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
- DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
- PROFITS, WHETHER IN AN ACTION OF CONTRACT, TORT (INCLUDING
- NEGLIGENCE) OR STRICT LIABILITY, ARISING OUT OF OR IN CONNECTION WITH
- THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
/* **********************************************************************
- Copyright 2005 Rensselaer Polytechnic Institute. All worldwide rights reserved.
+ Copyright 2008 Rensselaer Polytechnic Institute. All worldwide rights reserved.
Redistribution and use of this distribution in source and binary forms,
with or without modification, are permitted provided that:
@@ -59,13 +31,23 @@
import org.bedework.calsvc.indexing.BwIndexerFactory;
import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
/** Globals for the restore phase
*
- * @author Mike Douglass douglm at rpi.edu
+ * @author Mike Douglass douglm - rpi.edu
* @version 1.0
*/
-public class IndexerMap extends HashMap<BwUser, BwIndexer> {
+public class IndexerMap {
+ private static final int batchSize = 50;
+
+ private static class Indexer {
+ BwIndexer idx;
+ int batchCt;
+ }
+
+ private HashMap<BwUser, Indexer> map = new HashMap<BwUser, Indexer>();
private boolean debug;
private BwSystem syspars;
@@ -81,44 +63,43 @@
}
/**
- * @param val
- */
- public BwIndexer put(BwUser user, BwIndexer val) {
- if (get(user) != null) {
- throw new RuntimeException("User already in indexer table: " + user);
- }
-
- /* We could use a least recently used list of users here so we can keep the
- * size down by flushing old entries.
- */
-
- return super.put(user, val);
- }
-
- /**
* @param key BwUser
* @return BwIndexer
*/
public BwIndexer get(BwUser key) {
- BwIndexer indexer = super.get(key);
+ Indexer indexer = map.get(key);
if (indexer != null) {
- return indexer;
+ return indexer.idx;
}
/* We could build a least recently used list or users here so we can keep the
* size down.
*/
+ indexer = new Indexer();
+
try {
- indexer = BwIndexerFactory.getIndexer(false, key, true, syspars, debug);
+ indexer.idx = BwIndexerFactory.getIndexer(false, key, true, syspars, debug);
+ indexer.idx.setBatchSize(batchSize);
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException(t);
}
- super.put(key, indexer);
+ map.put(key, indexer);
- return indexer;
+ return indexer.idx;
}
+
+ /**
+ * @throws Throwable
+ */
+ public void flush() throws Throwable {
+ Set<Map.Entry<BwUser, Indexer>> entries = map.entrySet();
+
+ for (Map.Entry<BwUser, Indexer> entry: entries) {
+ entry.getValue().idx.flush();
+ }
+ }
}
Modified: releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/Restore.java
===================================================================
--- releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/Restore.java 2008-05-15 20:52:24 UTC (rev 203)
+++ releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/Restore.java 2008-05-21 18:52:29 UTC (rev 204)
@@ -99,6 +99,8 @@
}
globals.getPublicIndexer().flush();
globals.getPublicIndexer().endBwBatch();
+
+ globals.flushUserIndexers();
}
void doRestore() throws Throwable {
@@ -125,6 +127,8 @@
globals.config.setDebug(true);
} else if (args[i].equals("-ndebug")) {
globals.config.setDebug(false);
+ } else if (args[i].equals("-noindexes")) {
+ globals.noIndexes = true;
} else if (args[i].equals("")) {
// null arg generated by ant
} else if (args[i].equals("-noarg")) {
Modified: releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/RestoreGlobals.java
===================================================================
--- releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/RestoreGlobals.java 2008-05-15 20:52:24 UTC (rev 203)
+++ releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/RestoreGlobals.java 2008-05-21 18:52:29 UTC (rev 204)
@@ -41,7 +41,6 @@
import org.bedework.calfacade.exc.CalFacadeException;
import org.bedework.calfacade.filter.BwFilter;
import org.bedework.calfacade.svc.BwAdminGroup;
-//import org.bedework.calfacade.timezones.CalTimezones;
import org.bedework.calfacade.timezones.CalTimezones;
import org.bedework.calsvc.indexing.BwIndexer;
import org.bedework.calsvc.indexing.BwIndexerFactory;
@@ -91,6 +90,18 @@
*/
public boolean entityError;
+ /** True if we should skip lucene indexing
+ */
+ public boolean noIndexes;
+
+ /** Count time spent in indexer
+ */
+ public long indexTime;
+
+ /** Time spent in indexer at start of batch
+ */
+ public long indexTimeBatchStart;
+
/** Config properties from options file.
*/
public DumpRestoreConfig config;
@@ -112,7 +123,7 @@
/** */
public int nextCalKey = 1;
- /** Even number of elements, old-name followd by new-name
+ /** Even number of elements, old-name followed by new-name
*/
public ArrayList<String> fixedCalendarNames = new ArrayList<String>();
@@ -642,6 +653,15 @@
return idx;
}
+ /**
+ * @throws Throwable
+ */
+ public void flushUserIndexers() throws Throwable {
+ if (userIndexers != null) {
+ userIndexers.flush();
+ }
+ }
+
/** Who we are pretending to be for the core classes
*/
public BwUser currentUser;
Modified: releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/RestoreIntf.java
===================================================================
--- releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/RestoreIntf.java 2008-05-15 20:52:24 UTC (rev 203)
+++ releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/RestoreIntf.java 2008-05-21 18:52:29 UTC (rev 204)
@@ -312,21 +312,6 @@
*/
public void restoreDbLastmod(BwDbLastmod o) throws Throwable;
- /** Update all events for given user and set cal as the default calendar.
- *
- * @param u
- * @param cal
- * @return int num updated
- * @throws Throwable
- */
- public int fixUserEventsCal(BwUser u, BwCalendar cal) throws Throwable;
-
- /** 2.3 only
- * @param val
- * @throws Throwable
- */
- public void restoreCalendars(BwCalendar val) throws Throwable;
-
/**
* @param path
* @return BwCalendar
Modified: releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/rules/CalendarRule.java
===================================================================
--- releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/rules/CalendarRule.java 2008-05-15 20:52:24 UTC (rev 203)
+++ releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/rules/CalendarRule.java 2008-05-21 18:52:29 UTC (rev 204)
@@ -76,6 +76,13 @@
globals.calendars++;
+ if ((globals.calendars % 100) == 0) {
+ info("Restore calendar # " + globals.calendars +
+ " Time in indexer last batch " +
+ (globals.indexTime - globals.indexTimeBatchStart));
+ globals.indexTimeBatchStart = globals.indexTime;
+ }
+
fixSharableEntity(entity, "Calendar");
if ((globals.fixCaltype || globals.skipSpecialCals) &&
Modified: releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/rules/EventRule.java
===================================================================
--- releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/rules/EventRule.java 2008-05-15 20:52:24 UTC (rev 203)
+++ releases/bedework-3.4.1.1/src/org/bedework/dumprestore/restore/rules/EventRule.java 2008-05-21 18:52:29 UTC (rev 204)
@@ -154,7 +154,10 @@
globals.events++;
if ((globals.events % 100) == 0) {
- info("Restore event # " + globals.events);
+ info("Restore event # " + globals.events +
+ " Time in indexer last batch " +
+ (globals.indexTime - globals.indexTimeBatchStart));
+ globals.indexTimeBatchStart = globals.indexTime;
}
if (!override) {
More information about the Bedework-commit
mailing list