From b2dd8bf148b70c3076625840f908b5c7d32e9ee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kjetil=20=C3=98rbekk?= Date: Fri, 3 Sep 2010 01:11:03 -0400 Subject: Make ImportQuote work on AppEngine. Saving the entire database in one request caused a timeout. Instead, save one quote (plus its votes) per request. --- src/lq/ImportQuotes.java | 68 ++++++++++++++++++++++++++++++------------------ src/lq/Quote.java | 8 ++++++ src/lq/Vote.java | 2 ++ 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/lq/ImportQuotes.java b/src/lq/ImportQuotes.java index 7bfb5d8..bda4a8f 100644 --- a/src/lq/ImportQuotes.java +++ b/src/lq/ImportQuotes.java @@ -3,6 +3,7 @@ package lq; import java.io.File; import java.io.IOException; import java.util.Scanner; +import java.util.HashMap; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -22,13 +23,13 @@ import java.util.logging.Logger; public class ImportQuotes extends HttpServlet { private static final Logger logger = Logger.getLogger(ImportQuotes.class.getName()); - private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - private static final SimpleDateFormat timestampFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + // hacks.... + private HashMap quoteCache; + @Override public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - + quoteCache = new HashMap(); String path = "db-dump.xml"; resp.setContentType("text/plain"); @@ -45,20 +46,40 @@ public class ImportQuotes extends HttpServlet { "//table_data[@name='votes']/*"); List quotes = importQuotes(quoteRows); - List votes = importVotes(quotes, voteRows); + List votes = importVotes(voteRows); PersistenceManager pm = PMF.get().getPersistenceManager(); try { - pm.makePersistentAll(quotes); - pm.makePersistentAll(votes); + String importIdString = req.getParameter("importId"); + if (importIdString == null) { + uploadInChunks(pm, votes, 100); + uploadInChunks(pm, quotes, 100); + resp.getWriter().println("Import finished."); + } + else { + int importId = Integer.parseInt(importIdString); + pm.makePersistent(quotes.get(importId)); + resp.getWriter().println("There are " + quotes.size() + + " quotes in the old db."); + resp.getWriter().println("Quote " + importId + " imported."); + } } finally { pm.close(); } } catch (Exception e) { - resp.getWriter().println("Failed to parse xml: " + e.toString()); - e.printStackTrace(); + e.printStackTrace(resp.getWriter()); + } + } + + private void uploadInChunks(PersistenceManager pm, List objects, int chunkSize) { + for (int i = 0; i < objects.size(); i += chunkSize) { + List chunk = new ArrayList(); + for (int j = i; j < Math.min(objects.size(), i + chunkSize); j++) { + chunk.add(objects.get(j)); + } + pm.makePersistentAll(chunk); } } @@ -74,7 +95,9 @@ public class ImportQuotes extends HttpServlet { } private Quote importQuote(Element quoteRow) throws Exception { - Long id = Long.parseLong(getChildWithName("id", quoteRow)); + Long oldId = Long.parseLong(getChildWithName("id", quoteRow)); + + Long id = Long.parseLong(getChildWithName("number", quoteRow)); String approvedString = getChildWithName("approved", quoteRow); Boolean approved = approvedString.equals("1") ? true : false; @@ -82,12 +105,12 @@ public class ImportQuotes extends HttpServlet { String name = getChildWithName("name", quoteRow); String dateString = getChildWithName("date", quoteRow); - Date date = dateFormat.parse(dateString); + Date date = DateUtil.dateFormat.parse(dateString); String content = getChildWithName("text", quoteRow); String timeString = getChildWithName("time", quoteRow); - Date timestamp = timestampFormat.parse(timeString); + Date timestamp = DateUtil.timestampFormat.parse(timeString); String ip = getChildWithName("ip", quoteRow); @@ -95,15 +118,17 @@ public class ImportQuotes extends HttpServlet { quote.setId(id); quote.setTimestamp(timestamp); quote.setApproved(approved); + + quoteCache.put(oldId, quote); return quote; } - private List importVotes(List quotes, List voteRows) throws Exception { + private List importVotes(List voteRows) throws Exception { List votes = new ArrayList(); for (Object row : voteRows) { @SuppressWarnings("unchecked") Element elem = (Element) row; - Vote vote = importVote(elem, quotes); + Vote vote = importVote(elem); if (vote != null) { votes.add(vote); } @@ -112,15 +137,15 @@ public class ImportQuotes extends HttpServlet { return votes; } - private Vote importVote(Element voteRow, List quotes) throws Exception { + private Vote importVote(Element voteRow) throws Exception { Long quoteId = Long.parseLong(getChildWithName("id", voteRow)); - Quote quote = getQuoteWithId(quotes, quoteId); + Quote quote = quoteCache.get(quoteId); if (quote == null) { logger.warning("Could not find quote with id " + quoteId); return null; } Long rating = Long.parseLong(getChildWithName("vote", voteRow)); - Date timestamp = timestampFormat.parse(getChildWithName("time", voteRow)); + Date timestamp = lq.DateUtil.timestampFormat.parse(getChildWithName("time", voteRow)); String ip = getChildWithName("ip", voteRow); Vote vote = new Vote(rating, ip); @@ -134,14 +159,5 @@ public class ImportQuotes extends HttpServlet { Element child = (Element) XPath.selectSingleNode(parent, xpath); return child.getValue(); } - - private Quote getQuoteWithId(List quotes, Long id) { - for (Quote quote : quotes) { - if (quote.getId().equals(id)) { - return quote; - } - } - throw new RuntimeException("Could not find quote."); - } } diff --git a/src/lq/Quote.java b/src/lq/Quote.java index b0595b7..8edefaa 100644 --- a/src/lq/Quote.java +++ b/src/lq/Quote.java @@ -51,6 +51,14 @@ public class Quote { this.timestamp = new Date(); } + public Double getScore() { + Double score = 0.0; + for (Vote vote : getVotes()) { + score = score + vote.getRating() / getVotes().size(); + } + return score; + } + public Key getKey() { return key; } public Long getId() { return id; } public Date getTimestamp() { return timestamp; } diff --git a/src/lq/Vote.java b/src/lq/Vote.java index 9b9dca1..65bdf6c 100644 --- a/src/lq/Vote.java +++ b/src/lq/Vote.java @@ -37,9 +37,11 @@ public class Vote { public Quote getQuote() { return quote; } public Date getTimestamp() { return timestamp; } public String getIp() { return ip; } + public Long getRating() { return rating; } public void setKey(Key key) { this.key = key; } public void setQuote(Quote quote) { this.quote = quote; } public void setTimestamp(Date timestamp) { this.timestamp = timestamp; } public void setIp(String ip) { this.ip = ip; } + public void setRating(Long rating) { this.rating = rating; } } -- cgit v1.2.3