1 | /* |
2 | * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0, |
3 | * and the EPL 1.0 (http://h2database.com/html/license.html). |
4 | * Initial Developer: H2 Group |
5 | */ |
6 | package org.h2.tools; |
7 | |
8 | import java.io.BufferedInputStream; |
9 | import java.io.BufferedReader; |
10 | import java.io.ByteArrayInputStream; |
11 | import java.io.IOException; |
12 | import java.io.InputStream; |
13 | import java.io.InputStreamReader; |
14 | import java.io.OutputStream; |
15 | import java.io.PrintWriter; |
16 | import java.io.Reader; |
17 | import java.io.SequenceInputStream; |
18 | import java.sql.Connection; |
19 | import java.sql.PreparedStatement; |
20 | import java.sql.ResultSet; |
21 | import java.sql.SQLException; |
22 | import java.util.ArrayList; |
23 | import java.util.Collections; |
24 | import java.util.Enumeration; |
25 | import java.util.HashMap; |
26 | import java.util.HashSet; |
27 | import java.util.Iterator; |
28 | import java.util.Map; |
29 | import java.util.Map.Entry; |
30 | import java.util.zip.CRC32; |
31 | import org.h2.api.JavaObjectSerializer; |
32 | import org.h2.compress.CompressLZF; |
33 | import org.h2.engine.Constants; |
34 | import org.h2.engine.DbObject; |
35 | import org.h2.engine.MetaRecord; |
36 | import org.h2.jdbc.JdbcConnection; |
37 | import org.h2.message.DbException; |
38 | import org.h2.mvstore.MVMap; |
39 | import org.h2.mvstore.MVStore; |
40 | import org.h2.mvstore.MVStoreTool; |
41 | import org.h2.mvstore.StreamStore; |
42 | import org.h2.mvstore.db.TransactionStore; |
43 | import org.h2.mvstore.db.TransactionStore.TransactionMap; |
44 | import org.h2.mvstore.db.ValueDataType; |
45 | import org.h2.result.Row; |
46 | import org.h2.result.SimpleRow; |
47 | import org.h2.security.SHA256; |
48 | import org.h2.store.Data; |
49 | import org.h2.store.DataHandler; |
50 | import org.h2.store.DataReader; |
51 | import org.h2.store.FileLister; |
52 | import org.h2.store.FileStore; |
53 | import org.h2.store.FileStoreInputStream; |
54 | import org.h2.store.LobStorageBackend; |
55 | import org.h2.store.LobStorageFrontend; |
56 | import org.h2.store.Page; |
57 | import org.h2.store.PageFreeList; |
58 | import org.h2.store.PageLog; |
59 | import org.h2.store.PageStore; |
60 | import org.h2.store.fs.FileUtils; |
61 | import org.h2.util.BitField; |
62 | import org.h2.util.IOUtils; |
63 | import org.h2.util.IntArray; |
64 | import org.h2.util.MathUtils; |
65 | import org.h2.util.New; |
66 | import org.h2.util.SmallLRUCache; |
67 | import org.h2.util.StatementBuilder; |
68 | import org.h2.util.StringUtils; |
69 | import org.h2.util.TempFileDeleter; |
70 | import org.h2.util.Tool; |
71 | import org.h2.util.Utils; |
72 | import org.h2.value.Value; |
73 | import org.h2.value.ValueArray; |
74 | import org.h2.value.ValueLob; |
75 | import org.h2.value.ValueLobDb; |
76 | import org.h2.value.ValueLong; |
77 | |
78 | /** |
79 | * Helps recovering a corrupted database. |
80 | * @h2.resource |
81 | */ |
82 | public class Recover extends Tool implements DataHandler { |
83 | |
84 | private String databaseName; |
85 | private int storageId; |
86 | private String storageName; |
87 | private int recordLength; |
88 | private int valueId; |
89 | private boolean trace; |
90 | private boolean transactionLog; |
91 | private ArrayList<MetaRecord> schema; |
92 | private HashSet<Integer> objectIdSet; |
93 | private HashMap<Integer, String> tableMap; |
94 | private HashMap<String, String> columnTypeMap; |
95 | private boolean remove; |
96 | |
97 | private int pageSize; |
98 | private FileStore store; |
99 | private int[] parents; |
100 | |
101 | private Stats stat; |
102 | private boolean lobMaps; |
103 | |
104 | /** |
105 | * Statistic data |
106 | */ |
107 | static class Stats { |
108 | |
109 | /** |
110 | * The empty space in bytes in a data leaf pages. |
111 | */ |
112 | long pageDataEmpty; |
113 | |
114 | /** |
115 | * The number of bytes used for data. |
116 | */ |
117 | long pageDataRows; |
118 | |
119 | /** |
120 | * The number of bytes used for the page headers. |
121 | */ |
122 | long pageDataHead; |
123 | |
124 | /** |
125 | * The count per page type. |
126 | */ |
127 | final int[] pageTypeCount = new int[Page.TYPE_STREAM_DATA + 2]; |
128 | |
129 | /** |
130 | * The number of free pages. |
131 | */ |
132 | int free; |
133 | } |
134 | |
135 | /** |
136 | * Options are case sensitive. Supported options are: |
137 | * <table> |
138 | * <tr><td>[-help] or [-?]</td> |
139 | * <td>Print the list of options</td></tr> |
140 | * <tr><td>[-dir <dir>]</td> |
141 | * <td>The directory (default: .)</td></tr> |
142 | * <tr><td>[-db <database>]</td> |
143 | * <td>The database name (all databases if not set)</td></tr> |
144 | * <tr><td>[-trace]</td> |
145 | * <td>Print additional trace information</td></tr> |
146 | * <tr><td>[-transactionLog]</td> |
147 | * <td>Print the transaction log</td></tr> |
148 | * </table> |
149 | * Encrypted databases need to be decrypted first. |
150 | * @h2.resource |
151 | * |
152 | * @param args the command line arguments |
153 | */ |
154 | public static void main(String... args) throws SQLException { |
155 | new Recover().runTool(args); |
156 | } |
157 | |
158 | /** |
159 | * Dumps the contents of a database file to a human readable text file. This |
160 | * text file can be used to recover most of the data. This tool does not |
161 | * open the database and can be used even if the database files are |
162 | * corrupted. A database can get corrupted if there is a bug in the database |
163 | * engine or file system software, or if an application writes into the |
164 | * database file that doesn't understand the the file format, or if there is |
165 | * a hardware problem. |
166 | * |
167 | * @param args the command line arguments |
168 | */ |
169 | @Override |
170 | public void runTool(String... args) throws SQLException { |
171 | String dir = "."; |
172 | String db = null; |
173 | for (int i = 0; args != null && i < args.length; i++) { |
174 | String arg = args[i]; |
175 | if ("-dir".equals(arg)) { |
176 | dir = args[++i]; |
177 | } else if ("-db".equals(arg)) { |
178 | db = args[++i]; |
179 | } else if ("-removePassword".equals(arg)) { |
180 | remove = true; |
181 | } else if ("-trace".equals(arg)) { |
182 | trace = true; |
183 | } else if ("-transactionLog".equals(arg)) { |
184 | transactionLog = true; |
185 | } else if (arg.equals("-help") || arg.equals("-?")) { |
186 | showUsage(); |
187 | return; |
188 | } else { |
189 | showUsageAndThrowUnsupportedOption(arg); |
190 | } |
191 | } |
192 | process(dir, db); |
193 | } |
194 | |
195 | /** |
196 | * INTERNAL |
197 | */ |
198 | public static Reader readClob(String fileName) throws IOException { |
199 | return new BufferedReader(new InputStreamReader(readBlob(fileName), |
200 | Constants.UTF8)); |
201 | } |
202 | |
203 | /** |
204 | * INTERNAL |
205 | */ |
206 | public static InputStream readBlob(String fileName) throws IOException { |
207 | return new BufferedInputStream(FileUtils.newInputStream(fileName)); |
208 | } |
209 | |
210 | /** |
211 | * INTERNAL |
212 | */ |
213 | public static Value.ValueBlob readBlobDb(Connection conn, long lobId, |
214 | long precision) { |
215 | DataHandler h = ((JdbcConnection) conn).getSession().getDataHandler(); |
216 | return ValueLobDb.create(Value.BLOB, h, LobStorageFrontend.TABLE_TEMP, |
217 | lobId, null, precision); |
218 | } |
219 | |
220 | /** |
221 | * INTERNAL |
222 | */ |
223 | public static Value.ValueClob readClobDb(Connection conn, long lobId, |
224 | long precision) { |
225 | DataHandler h = ((JdbcConnection) conn).getSession().getDataHandler(); |
226 | return ValueLobDb.create(Value.CLOB, h, LobStorageFrontend.TABLE_TEMP, |
227 | lobId, null, precision); |
228 | } |
229 | |
230 | /** |
231 | * INTERNAL |
232 | */ |
233 | public static InputStream readBlobMap(Connection conn, long lobId, |
234 | long precision) throws SQLException { |
235 | final PreparedStatement prep = conn.prepareStatement( |
236 | "SELECT DATA FROM INFORMATION_SCHEMA.LOB_BLOCKS " + |
237 | "WHERE LOB_ID = ? AND SEQ = ? AND ? > 0"); |
238 | prep.setLong(1, lobId); |
239 | // precision is currently not really used, |
240 | // it is just to improve readability of the script |
241 | prep.setLong(3, precision); |
242 | return new SequenceInputStream( |
243 | new Enumeration<InputStream>() { |
244 | |
245 | private int seq; |
246 | private byte[] data = fetch(); |
247 | |
248 | private byte[] fetch() { |
249 | try { |
250 | prep.setInt(2, seq++); |
251 | ResultSet rs = prep.executeQuery(); |
252 | if (rs.next()) { |
253 | return rs.getBytes(1); |
254 | } |
255 | return null; |
256 | } catch (SQLException e) { |
257 | throw DbException.convert(e); |
258 | } |
259 | } |
260 | |
261 | @Override |
262 | public boolean hasMoreElements() { |
263 | return data != null; |
264 | } |
265 | |
266 | @Override |
267 | public InputStream nextElement() { |
268 | ByteArrayInputStream in = new ByteArrayInputStream(data); |
269 | data = fetch(); |
270 | return in; |
271 | } |
272 | } |
273 | ); |
274 | } |
275 | |
276 | /** |
277 | * INTERNAL |
278 | */ |
279 | public static Reader readClobMap(Connection conn, long lobId, long precision) |
280 | throws Exception { |
281 | InputStream in = readBlobMap(conn, lobId, precision); |
282 | return new BufferedReader(new InputStreamReader(in, Constants.UTF8)); |
283 | } |
284 | |
285 | private void trace(String message) { |
286 | if (trace) { |
287 | out.println(message); |
288 | } |
289 | } |
290 | |
291 | private void traceError(String message, Throwable t) { |
292 | out.println(message + ": " + t.toString()); |
293 | if (trace) { |
294 | t.printStackTrace(out); |
295 | } |
296 | } |
297 | |
298 | /** |
299 | * Dumps the contents of a database to a SQL script file. |
300 | * |
301 | * @param dir the directory |
302 | * @param db the database name (null for all databases) |
303 | */ |
304 | public static void execute(String dir, String db) throws SQLException { |
305 | try { |
306 | new Recover().process(dir, db); |
307 | } catch (DbException e) { |
308 | throw DbException.toSQLException(e); |
309 | } |
310 | } |
311 | |
312 | private void process(String dir, String db) { |
313 | ArrayList<String> list = FileLister.getDatabaseFiles(dir, db, true); |
314 | if (list.size() == 0) { |
315 | printNoDatabaseFilesFound(dir, db); |
316 | } |
317 | for (String fileName : list) { |
318 | if (fileName.endsWith(Constants.SUFFIX_PAGE_FILE)) { |
319 | dumpPageStore(fileName); |
320 | } else if (fileName.endsWith(Constants.SUFFIX_LOB_FILE)) { |
321 | dumpLob(fileName, false); |
322 | } else if (fileName.endsWith(Constants.SUFFIX_MV_FILE)) { |
323 | String f = fileName.substring(0, fileName.length() - |
324 | Constants.SUFFIX_PAGE_FILE.length()); |
325 | PrintWriter writer; |
326 | writer = getWriter(fileName, ".txt"); |
327 | MVStoreTool.dump(fileName, writer, true); |
328 | MVStoreTool.info(fileName, writer); |
329 | writer.close(); |
330 | writer = getWriter(f + ".h2.db", ".sql"); |
331 | dumpMVStoreFile(writer, fileName); |
332 | writer.close(); |
333 | } |
334 | } |
335 | } |
336 | |
337 | private PrintWriter getWriter(String fileName, String suffix) { |
338 | fileName = fileName.substring(0, fileName.length() - 3); |
339 | String outputFile = fileName + suffix; |
340 | trace("Created file: " + outputFile); |
341 | try { |
342 | return new PrintWriter(IOUtils.getBufferedWriter( |
343 | FileUtils.newOutputStream(outputFile, false))); |
344 | } catch (IOException e) { |
345 | throw DbException.convertIOException(e, null); |
346 | } |
347 | } |
348 | |
349 | private void writeDataError(PrintWriter writer, String error, byte[] data) { |
350 | writer.println("-- ERROR: " + error + " storageId: " |
351 | + storageId + " recordLength: " + recordLength + " valueId: " + valueId); |
352 | StringBuilder sb = new StringBuilder(); |
353 | for (int i = 0; i < data.length; i++) { |
354 | int x = data[i] & 0xff; |
355 | if (x >= ' ' && x < 128) { |
356 | sb.append((char) x); |
357 | } else { |
358 | sb.append('?'); |
359 | } |
360 | } |
361 | writer.println("-- dump: " + sb.toString()); |
362 | sb = new StringBuilder(); |
363 | for (int i = 0; i < data.length; i++) { |
364 | int x = data[i] & 0xff; |
365 | sb.append(' '); |
366 | if (x < 16) { |
367 | sb.append('0'); |
368 | } |
369 | sb.append(Integer.toHexString(x)); |
370 | } |
371 | writer.println("-- dump: " + sb.toString()); |
372 | } |
373 | |
374 | private void dumpLob(String fileName, boolean lobCompression) { |
375 | OutputStream fileOut = null; |
376 | FileStore fileStore = null; |
377 | long size = 0; |
378 | String n = fileName + (lobCompression ? ".comp" : "") + ".txt"; |
379 | InputStream in = null; |
380 | try { |
381 | fileOut = FileUtils.newOutputStream(n, false); |
382 | fileStore = FileStore.open(null, fileName, "r"); |
383 | fileStore.init(); |
384 | in = new FileStoreInputStream(fileStore, this, lobCompression, false); |
385 | size = IOUtils.copy(in, fileOut); |
386 | } catch (Throwable e) { |
387 | // this is usually not a problem, because we try both compressed and |
388 | // uncompressed |
389 | } finally { |
390 | IOUtils.closeSilently(fileOut); |
391 | IOUtils.closeSilently(in); |
392 | closeSilently(fileStore); |
393 | } |
394 | if (size == 0) { |
395 | try { |
396 | FileUtils.delete(n); |
397 | } catch (Exception e) { |
398 | traceError(n, e); |
399 | } |
400 | } |
401 | } |
402 | |
403 | private String getSQL(String column, Value v) { |
404 | if (v instanceof ValueLob) { |
405 | ValueLob lob = (ValueLob) v; |
406 | byte[] small = lob.getSmall(); |
407 | if (small == null) { |
408 | String file = lob.getFileName(); |
409 | String type = lob.getType() == Value.BLOB ? "BLOB" : "CLOB"; |
410 | if (lob.isCompressed()) { |
411 | dumpLob(file, true); |
412 | file += ".comp"; |
413 | } |
414 | return "READ_" + type + "('" + file + ".txt')"; |
415 | } |
416 | } else if (v instanceof ValueLobDb) { |
417 | ValueLobDb lob = (ValueLobDb) v; |
418 | byte[] small = lob.getSmall(); |
419 | if (small == null) { |
420 | int type = lob.getType(); |
421 | long id = lob.getLobId(); |
422 | long precision = lob.getPrecision(); |
423 | String m; |
424 | String columnType; |
425 | if (type == Value.BLOB) { |
426 | columnType = "BLOB"; |
427 | m = "READ_BLOB"; |
428 | } else { |
429 | columnType = "CLOB"; |
430 | m = "READ_CLOB"; |
431 | } |
432 | if (lobMaps) { |
433 | m += "_MAP"; |
434 | } else { |
435 | m += "_DB"; |
436 | } |
437 | columnTypeMap.put(column, columnType); |
438 | return m + "(" + id + ", " + precision + ")"; |
439 | } |
440 | } |
441 | return v.getSQL(); |
442 | } |
443 | |
444 | private void setDatabaseName(String name) { |
445 | databaseName = name; |
446 | } |
447 | |
448 | private void dumpPageStore(String fileName) { |
449 | setDatabaseName(fileName.substring(0, fileName.length() - |
450 | Constants.SUFFIX_PAGE_FILE.length())); |
451 | PrintWriter writer = null; |
452 | stat = new Stats(); |
453 | try { |
454 | writer = getWriter(fileName, ".sql"); |
455 | writer.println("CREATE ALIAS IF NOT EXISTS READ_BLOB FOR \"" + |
456 | this.getClass().getName() + ".readBlob\";"); |
457 | writer.println("CREATE ALIAS IF NOT EXISTS READ_CLOB FOR \"" + |
458 | this.getClass().getName() + ".readClob\";"); |
459 | writer.println("CREATE ALIAS IF NOT EXISTS READ_BLOB_DB FOR \"" + |
460 | this.getClass().getName() + ".readBlobDb\";"); |
461 | writer.println("CREATE ALIAS IF NOT EXISTS READ_CLOB_DB FOR \"" + |
462 | this.getClass().getName() + ".readClobDb\";"); |
463 | resetSchema(); |
464 | store = FileStore.open(null, fileName, remove ? "rw" : "r"); |
465 | long length = store.length(); |
466 | try { |
467 | store.init(); |
468 | } catch (Exception e) { |
469 | writeError(writer, e); |
470 | } |
471 | Data s = Data.create(this, 128); |
472 | seek(0); |
473 | store.readFully(s.getBytes(), 0, 128); |
474 | s.setPos(48); |
475 | pageSize = s.readInt(); |
476 | int writeVersion = s.readByte(); |
477 | int readVersion = s.readByte(); |
478 | writer.println("-- pageSize: " + pageSize + |
479 | " writeVersion: " + writeVersion + |
480 | " readVersion: " + readVersion); |
481 | if (pageSize < PageStore.PAGE_SIZE_MIN || |
482 | pageSize > PageStore.PAGE_SIZE_MAX) { |
483 | pageSize = Constants.DEFAULT_PAGE_SIZE; |
484 | writer.println("-- ERROR: page size; using " + pageSize); |
485 | } |
486 | long pageCount = length / pageSize; |
487 | parents = new int[(int) pageCount]; |
488 | s = Data.create(this, pageSize); |
489 | for (long i = 3; i < pageCount; i++) { |
490 | s.reset(); |
491 | seek(i); |
492 | store.readFully(s.getBytes(), 0, 32); |
493 | s.readByte(); |
494 | s.readShortInt(); |
495 | parents[(int) i] = s.readInt(); |
496 | } |
497 | int logKey = 0, logFirstTrunkPage = 0, logFirstDataPage = 0; |
498 | s = Data.create(this, pageSize); |
499 | for (long i = 1;; i++) { |
500 | if (i == 3) { |
501 | break; |
502 | } |
503 | s.reset(); |
504 | seek(i); |
505 | store.readFully(s.getBytes(), 0, pageSize); |
506 | CRC32 crc = new CRC32(); |
507 | crc.update(s.getBytes(), 4, pageSize - 4); |
508 | int expected = (int) crc.getValue(); |
509 | int got = s.readInt(); |
510 | long writeCounter = s.readLong(); |
511 | int key = s.readInt(); |
512 | int firstTrunkPage = s.readInt(); |
513 | int firstDataPage = s.readInt(); |
514 | if (expected == got) { |
515 | logKey = key; |
516 | logFirstTrunkPage = firstTrunkPage; |
517 | logFirstDataPage = firstDataPage; |
518 | } |
519 | writer.println("-- head " + i + |
520 | ": writeCounter: " + writeCounter + |
521 | " log " + key + ":" + firstTrunkPage + "/" + firstDataPage + |
522 | " crc " + got + " (" + (expected == got ? |
523 | "ok" : ("expected: " + expected)) + ")"); |
524 | } |
525 | writer.println("-- log " + logKey + ":" + logFirstTrunkPage + |
526 | "/" + logFirstDataPage); |
527 | |
528 | PrintWriter devNull = new PrintWriter(new OutputStream() { |
529 | @Override |
530 | public void write(int b) { |
531 | // ignore |
532 | } |
533 | }); |
534 | dumpPageStore(devNull, pageCount); |
535 | stat = new Stats(); |
536 | schema.clear(); |
537 | objectIdSet = New.hashSet(); |
538 | dumpPageStore(writer, pageCount); |
539 | writeSchema(writer); |
540 | try { |
541 | dumpPageLogStream(writer, logKey, logFirstTrunkPage, |
542 | logFirstDataPage, pageCount); |
543 | } catch (IOException e) { |
544 | // ignore |
545 | } |
546 | writer.println("---- Statistics ----"); |
547 | writer.println("-- page count: " + pageCount + ", free: " + stat.free); |
548 | long total = Math.max(1, stat.pageDataRows + |
549 | stat.pageDataEmpty + stat.pageDataHead); |
550 | writer.println("-- page data bytes: head " + stat.pageDataHead + |
551 | ", empty " + stat.pageDataEmpty + |
552 | ", rows " + stat.pageDataRows + |
553 | " (" + (100 - 100L * stat.pageDataEmpty / total) + "% full)"); |
554 | for (int i = 0; i < stat.pageTypeCount.length; i++) { |
555 | int count = stat.pageTypeCount[i]; |
556 | if (count > 0) { |
557 | writer.println("-- " + getPageType(i) + " " + |
558 | (100 * count / pageCount) + "%, " + count + " page(s)"); |
559 | } |
560 | } |
561 | writer.close(); |
562 | } catch (Throwable e) { |
563 | writeError(writer, e); |
564 | } finally { |
565 | IOUtils.closeSilently(writer); |
566 | closeSilently(store); |
567 | } |
568 | } |
569 | |
570 | private void dumpMVStoreFile(PrintWriter writer, String fileName) { |
571 | writer.println("-- MVStore"); |
572 | writer.println("CREATE ALIAS IF NOT EXISTS READ_BLOB FOR \"" + |
573 | this.getClass().getName() + ".readBlob\";"); |
574 | writer.println("CREATE ALIAS IF NOT EXISTS READ_CLOB FOR \"" + |
575 | this.getClass().getName() + ".readClob\";"); |
576 | writer.println("CREATE ALIAS IF NOT EXISTS READ_BLOB_DB FOR \"" + |
577 | this.getClass().getName() + ".readBlobDb\";"); |
578 | writer.println("CREATE ALIAS IF NOT EXISTS READ_CLOB_DB FOR \"" + |
579 | this.getClass().getName() + ".readClobDb\";"); |
580 | writer.println("CREATE ALIAS IF NOT EXISTS READ_BLOB_MAP FOR \"" + |
581 | this.getClass().getName() + ".readBlobMap\";"); |
582 | writer.println("CREATE ALIAS IF NOT EXISTS READ_CLOB_MAP FOR \"" + |
583 | this.getClass().getName() + ".readClobMap\";"); |
584 | resetSchema(); |
585 | setDatabaseName(fileName.substring(0, fileName.length() - |
586 | Constants.SUFFIX_MV_FILE.length())); |
587 | MVStore mv = new MVStore.Builder(). |
588 | fileName(fileName).readOnly().open(); |
589 | dumpLobMaps(writer, mv); |
590 | writer.println("-- Meta"); |
591 | dumpMeta(writer, mv); |
592 | writer.println("-- Tables"); |
593 | TransactionStore store = new TransactionStore(mv); |
594 | try { |
595 | store.init(); |
596 | } catch (Throwable e) { |
597 | writeError(writer, e); |
598 | } |
599 | try { |
600 | for (String mapName : mv.getMapNames()) { |
601 | if (!mapName.startsWith("table.")) { |
602 | continue; |
603 | } |
604 | String tableId = mapName.substring("table.".length()); |
605 | ValueDataType keyType = new ValueDataType( |
606 | null, this, null); |
607 | ValueDataType valueType = new ValueDataType( |
608 | null, this, null); |
609 | TransactionMap<Value, Value> dataMap = store.begin().openMap( |
610 | mapName, keyType, valueType); |
611 | Iterator<Value> dataIt = dataMap.keyIterator(null); |
612 | boolean init = false; |
613 | while (dataIt.hasNext()) { |
614 | Value rowId = dataIt.next(); |
615 | Value[] values = ((ValueArray) dataMap.get(rowId)).getList(); |
616 | recordLength = values.length; |
617 | if (!init) { |
618 | setStorage(Integer.parseInt(tableId)); |
619 | // init the column types |
620 | for (valueId = 0; valueId < recordLength; valueId++) { |
621 | String columnName = storageName + "." + valueId; |
622 | getSQL(columnName, values[valueId]); |
623 | } |
624 | createTemporaryTable(writer); |
625 | init = true; |
626 | } |
627 | StringBuilder buff = new StringBuilder(); |
628 | buff.append("INSERT INTO O_").append(tableId) |
629 | .append(" VALUES("); |
630 | for (valueId = 0; valueId < recordLength; valueId++) { |
631 | if (valueId > 0) { |
632 | buff.append(", "); |
633 | } |
634 | String columnName = storageName + "." + valueId; |
635 | buff.append(getSQL(columnName, values[valueId])); |
636 | } |
637 | buff.append(");"); |
638 | writer.println(buff.toString()); |
639 | if (storageId == 0) { |
640 | try { |
641 | SimpleRow r = new SimpleRow(values); |
642 | MetaRecord meta = new MetaRecord(r); |
643 | schema.add(meta); |
644 | if (meta.getObjectType() == DbObject.TABLE_OR_VIEW) { |
645 | String sql = values[3].getString(); |
646 | String name = extractTableOrViewName(sql); |
647 | tableMap.put(meta.getId(), name); |
648 | } |
649 | } catch (Throwable t) { |
650 | writeError(writer, t); |
651 | } |
652 | } |
653 | } |
654 | } |
655 | writeSchema(writer); |
656 | writer.println("DROP ALIAS READ_BLOB_MAP;"); |
657 | writer.println("DROP ALIAS READ_CLOB_MAP;"); |
658 | writer.println("DROP TABLE IF EXISTS INFORMATION_SCHEMA.LOB_BLOCKS;"); |
659 | } catch (Throwable e) { |
660 | writeError(writer, e); |
661 | } finally { |
662 | mv.close(); |
663 | } |
664 | } |
665 | |
666 | private static void dumpMeta(PrintWriter writer, MVStore mv) { |
667 | MVMap<String, String> meta = mv.getMetaMap(); |
668 | for (Entry<String, String> e : meta.entrySet()) { |
669 | writer.println("-- " + e.getKey() + " = " + e.getValue()); |
670 | } |
671 | } |
672 | |
673 | private void dumpLobMaps(PrintWriter writer, MVStore mv) { |
674 | lobMaps = mv.hasMap("lobData"); |
675 | if (!lobMaps) { |
676 | return; |
677 | } |
678 | MVMap<Long, byte[]> lobData = mv.openMap("lobData"); |
679 | StreamStore streamStore = new StreamStore(lobData); |
680 | MVMap<Long, Object[]> lobMap = mv.openMap("lobMap"); |
681 | writer.println("-- LOB"); |
682 | writer.println("CREATE TABLE IF NOT EXISTS " + |
683 | "INFORMATION_SCHEMA.LOB_BLOCKS(" + |
684 | "LOB_ID BIGINT, SEQ INT, DATA BINARY, " + |
685 | "PRIMARY KEY(LOB_ID, SEQ));"); |
686 | for (Entry<Long, Object[]> e : lobMap.entrySet()) { |
687 | long lobId = e.getKey(); |
688 | Object[] value = e.getValue(); |
689 | byte[] streamStoreId = (byte[]) value[0]; |
690 | InputStream in = streamStore.get(streamStoreId); |
691 | int len = 8 * 1024; |
692 | byte[] block = new byte[len]; |
693 | try { |
694 | for (int seq = 0;; seq++) { |
695 | int l = IOUtils.readFully(in, block, block.length); |
696 | String x = StringUtils.convertBytesToHex(block, l); |
697 | if (l > 0) { |
698 | writer.println("INSERT INTO INFORMATION_SCHEMA.LOB_BLOCKS " + |
699 | "VALUES(" + lobId + ", " + seq + ", '" + x + "');"); |
700 | } |
701 | if (l != len) { |
702 | break; |
703 | } |
704 | } |
705 | } catch (IOException ex) { |
706 | writeError(writer, ex); |
707 | } |
708 | } |
709 | } |
710 | |
711 | private static String getPageType(int type) { |
712 | switch (type) { |
713 | case 0: |
714 | return "free"; |
715 | case Page.TYPE_DATA_LEAF: |
716 | return "data leaf"; |
717 | case Page.TYPE_DATA_NODE: |
718 | return "data node"; |
719 | case Page.TYPE_DATA_OVERFLOW: |
720 | return "data overflow"; |
721 | case Page.TYPE_BTREE_LEAF: |
722 | return "btree leaf"; |
723 | case Page.TYPE_BTREE_NODE: |
724 | return "btree node"; |
725 | case Page.TYPE_FREE_LIST: |
726 | return "free list"; |
727 | case Page.TYPE_STREAM_TRUNK: |
728 | return "stream trunk"; |
729 | case Page.TYPE_STREAM_DATA: |
730 | return "stream data"; |
731 | } |
732 | return "[" + type + "]"; |
733 | } |
734 | |
735 | private void dumpPageStore(PrintWriter writer, long pageCount) { |
736 | Data s = Data.create(this, pageSize); |
737 | for (long page = 3; page < pageCount; page++) { |
738 | s = Data.create(this, pageSize); |
739 | seek(page); |
740 | store.readFully(s.getBytes(), 0, pageSize); |
741 | dumpPage(writer, s, page, pageCount); |
742 | } |
743 | } |
744 | |
745 | private void dumpPage(PrintWriter writer, Data s, long page, long pageCount) { |
746 | try { |
747 | int type = s.readByte(); |
748 | switch (type) { |
749 | case Page.TYPE_EMPTY: |
750 | stat.pageTypeCount[type]++; |
751 | return; |
752 | } |
753 | boolean last = (type & Page.FLAG_LAST) != 0; |
754 | type &= ~Page.FLAG_LAST; |
755 | if (!PageStore.checksumTest(s.getBytes(), (int) page, pageSize)) { |
756 | writeDataError(writer, "checksum mismatch type: " + type, s.getBytes()); |
757 | } |
758 | s.readShortInt(); |
759 | switch (type) { |
760 | // type 1 |
761 | case Page.TYPE_DATA_LEAF: { |
762 | stat.pageTypeCount[type]++; |
763 | int parentPageId = s.readInt(); |
764 | setStorage(s.readVarInt()); |
765 | int columnCount = s.readVarInt(); |
766 | int entries = s.readShortInt(); |
767 | writer.println("-- page " + page + ": data leaf " + |
768 | (last ? "(last) " : "") + "parent: " + parentPageId + |
769 | " table: " + storageId + " entries: " + entries + |
770 | " columns: " + columnCount); |
771 | dumpPageDataLeaf(writer, s, last, page, columnCount, entries); |
772 | break; |
773 | } |
774 | // type 2 |
775 | case Page.TYPE_DATA_NODE: { |
776 | stat.pageTypeCount[type]++; |
777 | int parentPageId = s.readInt(); |
778 | setStorage(s.readVarInt()); |
779 | int rowCount = s.readInt(); |
780 | int entries = s.readShortInt(); |
781 | writer.println("-- page " + page + ": data node " + |
782 | (last ? "(last) " : "") + "parent: " + parentPageId + |
783 | " table: " + storageId + " entries: " + entries + |
784 | " rowCount: " + rowCount); |
785 | dumpPageDataNode(writer, s, page, entries); |
786 | break; |
787 | } |
788 | // type 3 |
789 | case Page.TYPE_DATA_OVERFLOW: |
790 | stat.pageTypeCount[type]++; |
791 | writer.println("-- page " + page + ": data overflow " + |
792 | (last ? "(last) " : "")); |
793 | break; |
794 | // type 4 |
795 | case Page.TYPE_BTREE_LEAF: { |
796 | stat.pageTypeCount[type]++; |
797 | int parentPageId = s.readInt(); |
798 | setStorage(s.readVarInt()); |
799 | int entries = s.readShortInt(); |
800 | writer.println("-- page " + page + ": b-tree leaf " + |
801 | (last ? "(last) " : "") + "parent: " + parentPageId + |
802 | " index: " + storageId + " entries: " + entries); |
803 | if (trace) { |
804 | dumpPageBtreeLeaf(writer, s, entries, !last); |
805 | } |
806 | break; |
807 | } |
808 | // type 5 |
809 | case Page.TYPE_BTREE_NODE: |
810 | stat.pageTypeCount[type]++; |
811 | int parentPageId = s.readInt(); |
812 | setStorage(s.readVarInt()); |
813 | writer.println("-- page " + page + ": b-tree node " + |
814 | (last ? "(last) " : "") + "parent: " + parentPageId + |
815 | " index: " + storageId); |
816 | dumpPageBtreeNode(writer, s, page, !last); |
817 | break; |
818 | // type 6 |
819 | case Page.TYPE_FREE_LIST: |
820 | stat.pageTypeCount[type]++; |
821 | writer.println("-- page " + page + ": free list " + (last ? "(last)" : "")); |
822 | stat.free += dumpPageFreeList(writer, s, page, pageCount); |
823 | break; |
824 | // type 7 |
825 | case Page.TYPE_STREAM_TRUNK: |
826 | stat.pageTypeCount[type]++; |
827 | writer.println("-- page " + page + ": log trunk"); |
828 | break; |
829 | // type 8 |
830 | case Page.TYPE_STREAM_DATA: |
831 | stat.pageTypeCount[type]++; |
832 | writer.println("-- page " + page + ": log data"); |
833 | break; |
834 | default: |
835 | writer.println("-- ERROR page " + page + " unknown type " + type); |
836 | break; |
837 | } |
838 | } catch (Exception e) { |
839 | writeError(writer, e); |
840 | } |
841 | } |
842 | |
843 | private void dumpPageLogStream(PrintWriter writer, int logKey, |
844 | int logFirstTrunkPage, int logFirstDataPage, long pageCount) |
845 | throws IOException { |
846 | Data s = Data.create(this, pageSize); |
847 | DataReader in = new DataReader( |
848 | new PageInputStream(writer, this, store, logKey, |
849 | logFirstTrunkPage, logFirstDataPage, pageSize) |
850 | ); |
851 | writer.println("---- Transaction log ----"); |
852 | CompressLZF compress = new CompressLZF(); |
853 | while (true) { |
854 | int x = in.readByte(); |
855 | if (x < 0) { |
856 | break; |
857 | } |
858 | if (x == PageLog.NOOP) { |
859 | // ignore |
860 | } else if (x == PageLog.UNDO) { |
861 | int pageId = in.readVarInt(); |
862 | int size = in.readVarInt(); |
863 | byte[] data = new byte[pageSize]; |
864 | if (size == 0) { |
865 | in.readFully(data, pageSize); |
866 | } else if (size == 1) { |
867 | // empty |
868 | } else { |
869 | byte[] compressBuffer = new byte[size]; |
870 | in.readFully(compressBuffer, size); |
871 | try { |
872 | compress.expand(compressBuffer, 0, size, data, 0, pageSize); |
873 | } catch (ArrayIndexOutOfBoundsException e) { |
874 | throw DbException.convertToIOException(e); |
875 | } |
876 | } |
877 | String typeName = ""; |
878 | int type = data[0]; |
879 | boolean last = (type & Page.FLAG_LAST) != 0; |
880 | type &= ~Page.FLAG_LAST; |
881 | switch (type) { |
882 | case Page.TYPE_EMPTY: |
883 | typeName = "empty"; |
884 | break; |
885 | case Page.TYPE_DATA_LEAF: |
886 | typeName = "data leaf " + (last ? "(last)" : ""); |
887 | break; |
888 | case Page.TYPE_DATA_NODE: |
889 | typeName = "data node " + (last ? "(last)" : ""); |
890 | break; |
891 | case Page.TYPE_DATA_OVERFLOW: |
892 | typeName = "data overflow " + (last ? "(last)" : ""); |
893 | break; |
894 | case Page.TYPE_BTREE_LEAF: |
895 | typeName = "b-tree leaf " + (last ? "(last)" : ""); |
896 | break; |
897 | case Page.TYPE_BTREE_NODE: |
898 | typeName = "b-tree node " + (last ? "(last)" : ""); |
899 | break; |
900 | case Page.TYPE_FREE_LIST: |
901 | typeName = "free list " + (last ? "(last)" : ""); |
902 | break; |
903 | case Page.TYPE_STREAM_TRUNK: |
904 | typeName = "log trunk"; |
905 | break; |
906 | case Page.TYPE_STREAM_DATA: |
907 | typeName = "log data"; |
908 | break; |
909 | default: |
910 | typeName = "ERROR: unknown type " + type; |
911 | break; |
912 | } |
913 | writer.println("-- undo page " + pageId + " " + typeName); |
914 | if (trace) { |
915 | Data d = Data.create(null, data); |
916 | dumpPage(writer, d, pageId, pageCount); |
917 | } |
918 | } else if (x == PageLog.ADD) { |
919 | int sessionId = in.readVarInt(); |
920 | setStorage(in.readVarInt()); |
921 | Row row = PageLog.readRow(in, s); |
922 | writer.println("-- session " + sessionId + |
923 | " table " + storageId + |
924 | " + " + row.toString()); |
925 | if (transactionLog) { |
926 | if (storageId == 0 && row.getColumnCount() >= 4) { |
927 | int tableId = (int) row.getKey(); |
928 | String sql = row.getValue(3).getString(); |
929 | String name = extractTableOrViewName(sql); |
930 | if (row.getValue(2).getInt() == DbObject.TABLE_OR_VIEW) { |
931 | tableMap.put(tableId, name); |
932 | } |
933 | writer.println(sql + ";"); |
934 | } else { |
935 | String tableName = tableMap.get(storageId); |
936 | if (tableName != null) { |
937 | StatementBuilder buff = new StatementBuilder(); |
938 | buff.append("INSERT INTO ").append(tableName). |
939 | append(" VALUES("); |
940 | for (int i = 0; i < row.getColumnCount(); i++) { |
941 | buff.appendExceptFirst(", "); |
942 | buff.append(row.getValue(i).getSQL()); |
943 | } |
944 | buff.append(");"); |
945 | writer.println(buff.toString()); |
946 | } |
947 | } |
948 | } |
949 | } else if (x == PageLog.REMOVE) { |
950 | int sessionId = in.readVarInt(); |
951 | setStorage(in.readVarInt()); |
952 | long key = in.readVarLong(); |
953 | writer.println("-- session " + sessionId + |
954 | " table " + storageId + |
955 | " - " + key); |
956 | if (transactionLog) { |
957 | if (storageId == 0) { |
958 | int tableId = (int) key; |
959 | String tableName = tableMap.get(tableId); |
960 | if (tableName != null) { |
961 | writer.println("DROP TABLE IF EXISTS " + tableName + ";"); |
962 | } |
963 | } else { |
964 | String tableName = tableMap.get(storageId); |
965 | if (tableName != null) { |
966 | String sql = "DELETE FROM " + tableName + |
967 | " WHERE _ROWID_ = " + key + ";"; |
968 | writer.println(sql); |
969 | } |
970 | } |
971 | } |
972 | } else if (x == PageLog.TRUNCATE) { |
973 | int sessionId = in.readVarInt(); |
974 | setStorage(in.readVarInt()); |
975 | writer.println("-- session " + sessionId + |
976 | " table " + storageId + |
977 | " truncate"); |
978 | if (transactionLog) { |
979 | writer.println("TRUNCATE TABLE " + storageId); |
980 | } |
981 | } else if (x == PageLog.COMMIT) { |
982 | int sessionId = in.readVarInt(); |
983 | writer.println("-- commit " + sessionId); |
984 | } else if (x == PageLog.ROLLBACK) { |
985 | int sessionId = in.readVarInt(); |
986 | writer.println("-- rollback " + sessionId); |
987 | } else if (x == PageLog.PREPARE_COMMIT) { |
988 | int sessionId = in.readVarInt(); |
989 | String transaction = in.readString(); |
990 | writer.println("-- prepare commit " + sessionId + " " + transaction); |
991 | } else if (x == PageLog.NOOP) { |
992 | // nothing to do |
993 | } else if (x == PageLog.CHECKPOINT) { |
994 | writer.println("-- checkpoint"); |
995 | } else if (x == PageLog.FREE_LOG) { |
996 | int size = in.readVarInt(); |
997 | StringBuilder buff = new StringBuilder("-- free"); |
998 | for (int i = 0; i < size; i++) { |
999 | buff.append(' ').append(in.readVarInt()); |
1000 | } |
1001 | writer.println(buff); |
1002 | } else { |
1003 | writer.println("-- ERROR: unknown operation " + x); |
1004 | break; |
1005 | } |
1006 | } |
1007 | } |
1008 | |
1009 | private String setStorage(int storageId) { |
1010 | this.storageId = storageId; |
1011 | this.storageName = "O_" + String.valueOf(storageId).replace('-', 'M'); |
1012 | return storageName; |
1013 | } |
1014 | |
1015 | /** |
1016 | * An input stream that reads the data from a page store. |
1017 | */ |
1018 | static class PageInputStream extends InputStream { |
1019 | |
1020 | private final PrintWriter writer; |
1021 | private final FileStore store; |
1022 | private final Data page; |
1023 | private final int pageSize; |
1024 | private long trunkPage; |
1025 | private long nextTrunkPage; |
1026 | private long dataPage; |
1027 | private final IntArray dataPages = new IntArray(); |
1028 | private boolean endOfFile; |
1029 | private int remaining; |
1030 | private int logKey; |
1031 | |
1032 | public PageInputStream(PrintWriter writer, DataHandler handler, |
1033 | FileStore store, int logKey, long firstTrunkPage, |
1034 | long firstDataPage, int pageSize) { |
1035 | this.writer = writer; |
1036 | this.store = store; |
1037 | this.pageSize = pageSize; |
1038 | this.logKey = logKey - 1; |
1039 | this.nextTrunkPage = firstTrunkPage; |
1040 | this.dataPage = firstDataPage; |
1041 | page = Data.create(handler, pageSize); |
1042 | } |
1043 | |
1044 | @Override |
1045 | public int read() { |
1046 | byte[] b = { 0 }; |
1047 | int len = read(b); |
1048 | return len < 0 ? -1 : (b[0] & 255); |
1049 | } |
1050 | |
1051 | @Override |
1052 | public int read(byte[] b) { |
1053 | return read(b, 0, b.length); |
1054 | } |
1055 | |
1056 | @Override |
1057 | public int read(byte[] b, int off, int len) { |
1058 | if (len == 0) { |
1059 | return 0; |
1060 | } |
1061 | int read = 0; |
1062 | while (len > 0) { |
1063 | int r = readBlock(b, off, len); |
1064 | if (r < 0) { |
1065 | break; |
1066 | } |
1067 | read += r; |
1068 | off += r; |
1069 | len -= r; |
1070 | } |
1071 | return read == 0 ? -1 : read; |
1072 | } |
1073 | |
1074 | private int readBlock(byte[] buff, int off, int len) { |
1075 | fillBuffer(); |
1076 | if (endOfFile) { |
1077 | return -1; |
1078 | } |
1079 | int l = Math.min(remaining, len); |
1080 | page.read(buff, off, l); |
1081 | remaining -= l; |
1082 | return l; |
1083 | } |
1084 | |
1085 | private void fillBuffer() { |
1086 | if (remaining > 0 || endOfFile) { |
1087 | return; |
1088 | } |
1089 | while (dataPages.size() == 0) { |
1090 | if (nextTrunkPage == 0) { |
1091 | endOfFile = true; |
1092 | return; |
1093 | } |
1094 | trunkPage = nextTrunkPage; |
1095 | store.seek(trunkPage * pageSize); |
1096 | store.readFully(page.getBytes(), 0, pageSize); |
1097 | page.reset(); |
1098 | if (!PageStore.checksumTest(page.getBytes(), (int) trunkPage, pageSize)) { |
1099 | writer.println("-- ERROR: checksum mismatch page: " +trunkPage); |
1100 | endOfFile = true; |
1101 | return; |
1102 | } |
1103 | int t = page.readByte(); |
1104 | page.readShortInt(); |
1105 | if (t != Page.TYPE_STREAM_TRUNK) { |
1106 | writer.println("-- log eof " + trunkPage + " type: " + t + |
1107 | " expected type: " + Page.TYPE_STREAM_TRUNK); |
1108 | endOfFile = true; |
1109 | return; |
1110 | } |
1111 | page.readInt(); |
1112 | int key = page.readInt(); |
1113 | logKey++; |
1114 | if (key != logKey) { |
1115 | writer.println("-- log eof " + trunkPage + |
1116 | " type: " + t + " expected key: " + logKey + " got: " + key); |
1117 | } |
1118 | nextTrunkPage = page.readInt(); |
1119 | writer.println("-- log " + key + ":" + trunkPage + |
1120 | " next: " + nextTrunkPage); |
1121 | int pageCount = page.readShortInt(); |
1122 | for (int i = 0; i < pageCount; i++) { |
1123 | int d = page.readInt(); |
1124 | if (dataPage != 0) { |
1125 | if (d == dataPage) { |
1126 | dataPage = 0; |
1127 | } else { |
1128 | // ignore the pages before the starting page |
1129 | continue; |
1130 | } |
1131 | } |
1132 | dataPages.add(d); |
1133 | } |
1134 | } |
1135 | if (dataPages.size() > 0) { |
1136 | page.reset(); |
1137 | long nextPage = dataPages.get(0); |
1138 | dataPages.remove(0); |
1139 | store.seek(nextPage * pageSize); |
1140 | store.readFully(page.getBytes(), 0, pageSize); |
1141 | page.reset(); |
1142 | int t = page.readByte(); |
1143 | if (t != 0 && !PageStore.checksumTest(page.getBytes(), |
1144 | (int) nextPage, pageSize)) { |
1145 | writer.println("-- ERROR: checksum mismatch page: " +nextPage); |
1146 | endOfFile = true; |
1147 | return; |
1148 | } |
1149 | page.readShortInt(); |
1150 | int p = page.readInt(); |
1151 | int k = page.readInt(); |
1152 | writer.println("-- log " + k + ":" + trunkPage + "/" + nextPage); |
1153 | if (t != Page.TYPE_STREAM_DATA) { |
1154 | writer.println("-- log eof " +nextPage+ " type: " + t + " parent: " + p + |
1155 | " expected type: " + Page.TYPE_STREAM_DATA); |
1156 | endOfFile = true; |
1157 | return; |
1158 | } else if (k != logKey) { |
1159 | writer.println("-- log eof " +nextPage+ " type: " + t + " parent: " + p + |
1160 | " expected key: " + logKey + " got: " + k); |
1161 | endOfFile = true; |
1162 | return; |
1163 | } |
1164 | remaining = pageSize - page.length(); |
1165 | } |
1166 | } |
1167 | } |
1168 | |
1169 | private void dumpPageBtreeNode(PrintWriter writer, Data s, long pageId, |
1170 | boolean positionOnly) { |
1171 | int rowCount = s.readInt(); |
1172 | int entryCount = s.readShortInt(); |
1173 | int[] children = new int[entryCount + 1]; |
1174 | int[] offsets = new int[entryCount]; |
1175 | children[entryCount] = s.readInt(); |
1176 | checkParent(writer, pageId, children, entryCount); |
1177 | int empty = Integer.MAX_VALUE; |
1178 | for (int i = 0; i < entryCount; i++) { |
1179 | children[i] = s.readInt(); |
1180 | checkParent(writer, pageId, children, i); |
1181 | int off = s.readShortInt(); |
1182 | empty = Math.min(off, empty); |
1183 | offsets[i] = off; |
1184 | } |
1185 | empty = empty - s.length(); |
1186 | if (!trace) { |
1187 | return; |
1188 | } |
1189 | writer.println("-- empty: " + empty); |
1190 | for (int i = 0; i < entryCount; i++) { |
1191 | int off = offsets[i]; |
1192 | s.setPos(off); |
1193 | long key = s.readVarLong(); |
1194 | Value data; |
1195 | if (positionOnly) { |
1196 | data = ValueLong.get(key); |
1197 | } else { |
1198 | try { |
1199 | data = s.readValue(); |
1200 | } catch (Throwable e) { |
1201 | writeDataError(writer, "exception " + e, s.getBytes()); |
1202 | continue; |
1203 | } |
1204 | } |
1205 | writer.println("-- [" + i + "] child: " + children[i] + |
1206 | " key: " + key + " data: " + data); |
1207 | } |
1208 | writer.println("-- [" + entryCount + "] child: " + |
1209 | children[entryCount] + " rowCount: " + rowCount); |
1210 | } |
1211 | |
1212 | private int dumpPageFreeList(PrintWriter writer, Data s, long pageId, |
1213 | long pageCount) { |
1214 | int pagesAddressed = PageFreeList.getPagesAddressed(pageSize); |
1215 | BitField used = new BitField(); |
1216 | for (int i = 0; i < pagesAddressed; i += 8) { |
1217 | int x = s.readByte() & 255; |
1218 | for (int j = 0; j < 8; j++) { |
1219 | if ((x & (1 << j)) != 0) { |
1220 | used.set(i + j); |
1221 | } |
1222 | } |
1223 | } |
1224 | int free = 0; |
1225 | for (long i = 0, j = pageId; i < pagesAddressed && j < pageCount; i++, j++) { |
1226 | if (i == 0 || j % 100 == 0) { |
1227 | if (i > 0) { |
1228 | writer.println(); |
1229 | } |
1230 | writer.print("-- " + j + " "); |
1231 | } else if (j % 20 == 0) { |
1232 | writer.print(" - "); |
1233 | } else if (j % 10 == 0) { |
1234 | writer.print(' '); |
1235 | } |
1236 | writer.print(used.get((int) i) ? '1' : '0'); |
1237 | if (!used.get((int) i)) { |
1238 | free++; |
1239 | } |
1240 | } |
1241 | writer.println(); |
1242 | return free; |
1243 | } |
1244 | |
1245 | private void dumpPageBtreeLeaf(PrintWriter writer, Data s, int entryCount, |
1246 | boolean positionOnly) { |
1247 | int[] offsets = new int[entryCount]; |
1248 | int empty = Integer.MAX_VALUE; |
1249 | for (int i = 0; i < entryCount; i++) { |
1250 | int off = s.readShortInt(); |
1251 | empty = Math.min(off, empty); |
1252 | offsets[i] = off; |
1253 | } |
1254 | empty = empty - s.length(); |
1255 | writer.println("-- empty: " + empty); |
1256 | for (int i = 0; i < entryCount; i++) { |
1257 | int off = offsets[i]; |
1258 | s.setPos(off); |
1259 | long key = s.readVarLong(); |
1260 | Value data; |
1261 | if (positionOnly) { |
1262 | data = ValueLong.get(key); |
1263 | } else { |
1264 | try { |
1265 | data = s.readValue(); |
1266 | } catch (Throwable e) { |
1267 | writeDataError(writer, "exception " + e, s.getBytes()); |
1268 | continue; |
1269 | } |
1270 | } |
1271 | writer.println("-- [" + i + "] key: " + key + " data: " + data); |
1272 | } |
1273 | } |
1274 | |
1275 | private void checkParent(PrintWriter writer, long pageId, int[] children, |
1276 | int index) { |
1277 | int child = children[index]; |
1278 | if (child < 0 || child >= parents.length) { |
1279 | writer.println("-- ERROR [" + pageId + "] child[" + |
1280 | index + "]: " + child + " >= page count: " + parents.length); |
1281 | } else if (parents[child] != pageId) { |
1282 | writer.println("-- ERROR [" + pageId + "] child[" + |
1283 | index + "]: " + child + " parent: " + parents[child]); |
1284 | } |
1285 | } |
1286 | |
1287 | private void dumpPageDataNode(PrintWriter writer, Data s, long pageId, |
1288 | int entryCount) { |
1289 | int[] children = new int[entryCount + 1]; |
1290 | long[] keys = new long[entryCount]; |
1291 | children[entryCount] = s.readInt(); |
1292 | checkParent(writer, pageId, children, entryCount); |
1293 | for (int i = 0; i < entryCount; i++) { |
1294 | children[i] = s.readInt(); |
1295 | checkParent(writer, pageId, children, i); |
1296 | keys[i] = s.readVarLong(); |
1297 | } |
1298 | if (!trace) { |
1299 | return; |
1300 | } |
1301 | for (int i = 0; i < entryCount; i++) { |
1302 | writer.println("-- [" + i + "] child: " + children[i] + " key: " + keys[i]); |
1303 | } |
1304 | writer.println("-- [" + entryCount + "] child: " + children[entryCount]); |
1305 | } |
1306 | |
1307 | private void dumpPageDataLeaf(PrintWriter writer, Data s, boolean last, |
1308 | long pageId, int columnCount, int entryCount) { |
1309 | long[] keys = new long[entryCount]; |
1310 | int[] offsets = new int[entryCount]; |
1311 | long next = 0; |
1312 | if (!last) { |
1313 | next = s.readInt(); |
1314 | writer.println("-- next: " + next); |
1315 | } |
1316 | int empty = pageSize; |
1317 | for (int i = 0; i < entryCount; i++) { |
1318 | keys[i] = s.readVarLong(); |
1319 | int off = s.readShortInt(); |
1320 | empty = Math.min(off, empty); |
1321 | offsets[i] = off; |
1322 | } |
1323 | stat.pageDataRows += pageSize - empty; |
1324 | empty = empty - s.length(); |
1325 | stat.pageDataHead += s.length(); |
1326 | stat.pageDataEmpty += empty; |
1327 | if (trace) { |
1328 | writer.println("-- empty: " + empty); |
1329 | } |
1330 | if (!last) { |
1331 | Data s2 = Data.create(this, pageSize); |
1332 | s.setPos(pageSize); |
1333 | long parent = pageId; |
1334 | while (true) { |
1335 | checkParent(writer, parent, new int[]{(int) next}, 0); |
1336 | parent = next; |
1337 | seek(next); |
1338 | store.readFully(s2.getBytes(), 0, pageSize); |
1339 | s2.reset(); |
1340 | int type = s2.readByte(); |
1341 | s2.readShortInt(); |
1342 | s2.readInt(); |
1343 | if (type == (Page.TYPE_DATA_OVERFLOW | Page.FLAG_LAST)) { |
1344 | int size = s2.readShortInt(); |
1345 | writer.println("-- chain: " + next + |
1346 | " type: " + type + " size: " + size); |
1347 | s.checkCapacity(size); |
1348 | s.write(s2.getBytes(), s2.length(), size); |
1349 | break; |
1350 | } else if (type == Page.TYPE_DATA_OVERFLOW) { |
1351 | next = s2.readInt(); |
1352 | if (next == 0) { |
1353 | writeDataError(writer, "next:0", s2.getBytes()); |
1354 | break; |
1355 | } |
1356 | int size = pageSize - s2.length(); |
1357 | writer.println("-- chain: " + next + " type: " + type + |
1358 | " size: " + size + " next: " + next); |
1359 | s.checkCapacity(size); |
1360 | s.write(s2.getBytes(), s2.length(), size); |
1361 | } else { |
1362 | writeDataError(writer, "type: " + type, s2.getBytes()); |
1363 | break; |
1364 | } |
1365 | } |
1366 | } |
1367 | for (int i = 0; i < entryCount; i++) { |
1368 | long key = keys[i]; |
1369 | int off = offsets[i]; |
1370 | if (trace) { |
1371 | writer.println("-- [" + i + "] storage: " + storageId + |
1372 | " key: " + key + " off: " + off); |
1373 | } |
1374 | s.setPos(off); |
1375 | Value[] data = createRecord(writer, s, columnCount); |
1376 | if (data != null) { |
1377 | createTemporaryTable(writer); |
1378 | writeRow(writer, s, data); |
1379 | if (remove && storageId == 0) { |
1380 | String sql = data[3].getString(); |
1381 | if (sql.startsWith("CREATE USER ")) { |
1382 | int saltIndex = Utils.indexOf(s.getBytes(), "SALT ".getBytes(), off); |
1383 | if (saltIndex >= 0) { |
1384 | String userName = sql.substring("CREATE USER ".length(), |
1385 | sql.indexOf("SALT ") - 1); |
1386 | if (userName.startsWith("IF NOT EXISTS ")) { |
1387 | userName = userName.substring("IF NOT EXISTS ".length()); |
1388 | } |
1389 | if (userName.startsWith("\"")) { |
1390 | // TODO doesn't work for all cases ("" inside |
1391 | // user name) |
1392 | userName = userName.substring(1, userName.length() - 1); |
1393 | } |
1394 | byte[] userPasswordHash = SHA256.getKeyPasswordHash( |
1395 | userName, "".toCharArray()); |
1396 | byte[] salt = MathUtils.secureRandomBytes(Constants.SALT_LEN); |
1397 | byte[] passwordHash = SHA256.getHashWithSalt( |
1398 | userPasswordHash, salt); |
1399 | StringBuilder buff = new StringBuilder(); |
1400 | buff.append("SALT '"). |
1401 | append(StringUtils.convertBytesToHex(salt)). |
1402 | append("' HASH '"). |
1403 | append(StringUtils.convertBytesToHex(passwordHash)). |
1404 | append('\''); |
1405 | byte[] replacement = buff.toString().getBytes(); |
1406 | System.arraycopy(replacement, 0, s.getBytes(), |
1407 | saltIndex, replacement.length); |
1408 | seek(pageId); |
1409 | store.write(s.getBytes(), 0, pageSize); |
1410 | if (trace) { |
1411 | out.println("User: " + userName); |
1412 | } |
1413 | remove = false; |
1414 | } |
1415 | } |
1416 | } |
1417 | } |
1418 | } |
1419 | } |
1420 | |
1421 | private void seek(long page) { |
1422 | // page is long to avoid integer overflow |
1423 | store.seek(page * pageSize); |
1424 | } |
1425 | |
1426 | private Value[] createRecord(PrintWriter writer, Data s, int columnCount) { |
1427 | recordLength = columnCount; |
1428 | if (columnCount <= 0) { |
1429 | writeDataError(writer, "columnCount<0", s.getBytes()); |
1430 | return null; |
1431 | } |
1432 | Value[] data; |
1433 | try { |
1434 | data = new Value[columnCount]; |
1435 | } catch (OutOfMemoryError e) { |
1436 | writeDataError(writer, "out of memory", s.getBytes()); |
1437 | return null; |
1438 | } |
1439 | return data; |
1440 | } |
1441 | |
1442 | private void writeRow(PrintWriter writer, Data s, Value[] data) { |
1443 | StringBuilder sb = new StringBuilder(); |
1444 | sb.append("INSERT INTO " + storageName + " VALUES("); |
1445 | for (valueId = 0; valueId < recordLength; valueId++) { |
1446 | try { |
1447 | Value v = s.readValue(); |
1448 | data[valueId] = v; |
1449 | if (valueId > 0) { |
1450 | sb.append(", "); |
1451 | } |
1452 | String columnName = storageName + "." + valueId; |
1453 | sb.append(getSQL(columnName, v)); |
1454 | } catch (Exception e) { |
1455 | writeDataError(writer, "exception " + e, s.getBytes()); |
1456 | continue; |
1457 | } catch (OutOfMemoryError e) { |
1458 | writeDataError(writer, "out of memory", s.getBytes()); |
1459 | continue; |
1460 | } |
1461 | } |
1462 | sb.append(");"); |
1463 | writer.println(sb.toString()); |
1464 | if (storageId == 0) { |
1465 | try { |
1466 | SimpleRow r = new SimpleRow(data); |
1467 | MetaRecord meta = new MetaRecord(r); |
1468 | schema.add(meta); |
1469 | if (meta.getObjectType() == DbObject.TABLE_OR_VIEW) { |
1470 | String sql = data[3].getString(); |
1471 | String name = extractTableOrViewName(sql); |
1472 | tableMap.put(meta.getId(), name); |
1473 | } |
1474 | } catch (Throwable t) { |
1475 | writeError(writer, t); |
1476 | } |
1477 | } |
1478 | } |
1479 | |
1480 | private void resetSchema() { |
1481 | schema = New.arrayList(); |
1482 | objectIdSet = New.hashSet(); |
1483 | tableMap = New.hashMap(); |
1484 | columnTypeMap = New.hashMap(); |
1485 | } |
1486 | |
1487 | private void writeSchema(PrintWriter writer) { |
1488 | writer.println("---- Schema ----"); |
1489 | Collections.sort(schema); |
1490 | for (MetaRecord m : schema) { |
1491 | if (!isSchemaObjectTypeDelayed(m)) { |
1492 | // create, but not referential integrity constraints and so on |
1493 | // because they could fail on duplicate keys |
1494 | String sql = m.getSQL(); |
1495 | writer.println(sql + ";"); |
1496 | } |
1497 | } |
1498 | // first, copy the lob storage (if there is any) |
1499 | // must occur before copying data, |
1500 | // otherwise the lob storage may be overwritten |
1501 | boolean deleteLobs = false; |
1502 | for (Map.Entry<Integer, String> entry : tableMap.entrySet()) { |
1503 | Integer objectId = entry.getKey(); |
1504 | String name = entry.getValue(); |
1505 | if (objectIdSet.contains(objectId)) { |
1506 | if (name.startsWith("INFORMATION_SCHEMA.LOB")) { |
1507 | setStorage(objectId); |
1508 | writer.println("DELETE FROM " + name + ";"); |
1509 | writer.println("INSERT INTO " + name + " SELECT * FROM " + storageName + ";"); |
1510 | if (name.startsWith("INFORMATION_SCHEMA.LOBS")) { |
1511 | writer.println("UPDATE " + name + " SET TABLE = " + |
1512 | LobStorageFrontend.TABLE_TEMP + ";"); |
1513 | deleteLobs = true; |
1514 | } |
1515 | } |
1516 | } |
1517 | } |
1518 | for (Map.Entry<Integer, String> entry : tableMap.entrySet()) { |
1519 | Integer objectId = entry.getKey(); |
1520 | String name = entry.getValue(); |
1521 | if (objectIdSet.contains(objectId)) { |
1522 | setStorage(objectId); |
1523 | if (name.startsWith("INFORMATION_SCHEMA.LOB")) { |
1524 | continue; |
1525 | } |
1526 | writer.println("INSERT INTO " + name + " SELECT * FROM " + storageName + ";"); |
1527 | } |
1528 | } |
1529 | for (Integer objectId : objectIdSet) { |
1530 | setStorage(objectId); |
1531 | writer.println("DROP TABLE " + storageName + ";"); |
1532 | } |
1533 | writer.println("DROP ALIAS READ_BLOB;"); |
1534 | writer.println("DROP ALIAS READ_CLOB;"); |
1535 | writer.println("DROP ALIAS READ_BLOB_DB;"); |
1536 | writer.println("DROP ALIAS READ_CLOB_DB;"); |
1537 | if (deleteLobs) { |
1538 | writer.println("DELETE FROM INFORMATION_SCHEMA.LOBS WHERE TABLE = " + |
1539 | LobStorageFrontend.TABLE_TEMP + ";"); |
1540 | } |
1541 | for (MetaRecord m : schema) { |
1542 | if (isSchemaObjectTypeDelayed(m)) { |
1543 | String sql = m.getSQL(); |
1544 | writer.println(sql + ";"); |
1545 | } |
1546 | } |
1547 | } |
1548 | |
1549 | private static boolean isSchemaObjectTypeDelayed(MetaRecord m) { |
1550 | switch (m.getObjectType()) { |
1551 | case DbObject.INDEX: |
1552 | case DbObject.CONSTRAINT: |
1553 | case DbObject.TRIGGER: |
1554 | return true; |
1555 | } |
1556 | return false; |
1557 | } |
1558 | |
1559 | private void createTemporaryTable(PrintWriter writer) { |
1560 | if (!objectIdSet.contains(storageId)) { |
1561 | objectIdSet.add(storageId); |
1562 | StatementBuilder buff = new StatementBuilder("CREATE TABLE "); |
1563 | buff.append(storageName).append('('); |
1564 | for (int i = 0; i < recordLength; i++) { |
1565 | buff.appendExceptFirst(", "); |
1566 | buff.append('C').append(i).append(' '); |
1567 | String columnType = columnTypeMap.get(storageName + "." + i); |
1568 | if (columnType == null) { |
1569 | buff.append("VARCHAR"); |
1570 | } else { |
1571 | buff.append(columnType); |
1572 | } |
1573 | } |
1574 | writer.println(buff.append(");").toString()); |
1575 | writer.flush(); |
1576 | } |
1577 | } |
1578 | |
1579 | private static String extractTableOrViewName(String sql) { |
1580 | int indexTable = sql.indexOf(" TABLE "); |
1581 | int indexView = sql.indexOf(" VIEW "); |
1582 | if (indexTable > 0 && indexView > 0) { |
1583 | if (indexTable < indexView) { |
1584 | indexView = -1; |
1585 | } else { |
1586 | indexTable = -1; |
1587 | } |
1588 | } |
1589 | if (indexView > 0) { |
1590 | sql = sql.substring(indexView + " VIEW ".length()); |
1591 | } else if (indexTable > 0) { |
1592 | sql = sql.substring(indexTable + " TABLE ".length()); |
1593 | } else { |
1594 | return "UNKNOWN"; |
1595 | } |
1596 | if (sql.startsWith("IF NOT EXISTS ")) { |
1597 | sql = sql.substring("IF NOT EXISTS ".length()); |
1598 | } |
1599 | boolean ignore = false; |
1600 | // sql is modified in the loop |
1601 | for (int i = 0; i < sql.length(); i++) { |
1602 | char ch = sql.charAt(i); |
1603 | if (ch == '\"') { |
1604 | ignore = !ignore; |
1605 | } else if (!ignore && (ch <= ' ' || ch == '(')) { |
1606 | sql = sql.substring(0, i); |
1607 | return sql; |
1608 | } |
1609 | } |
1610 | return "UNKNOWN"; |
1611 | } |
1612 | |
1613 | |
1614 | private static void closeSilently(FileStore fileStore) { |
1615 | if (fileStore != null) { |
1616 | fileStore.closeSilently(); |
1617 | } |
1618 | } |
1619 | |
1620 | private void writeError(PrintWriter writer, Throwable e) { |
1621 | if (writer != null) { |
1622 | writer.println("// error: " + e); |
1623 | } |
1624 | traceError("Error", e); |
1625 | } |
1626 | |
1627 | /** |
1628 | * INTERNAL |
1629 | */ |
1630 | @Override |
1631 | public String getDatabasePath() { |
1632 | return databaseName; |
1633 | } |
1634 | |
1635 | /** |
1636 | * INTERNAL |
1637 | */ |
1638 | @Override |
1639 | public FileStore openFile(String name, String mode, boolean mustExist) { |
1640 | return FileStore.open(this, name, "rw"); |
1641 | } |
1642 | |
1643 | /** |
1644 | * INTERNAL |
1645 | */ |
1646 | @Override |
1647 | public void checkPowerOff() { |
1648 | // nothing to do |
1649 | } |
1650 | |
1651 | /** |
1652 | * INTERNAL |
1653 | */ |
1654 | @Override |
1655 | public void checkWritingAllowed() { |
1656 | // nothing to do |
1657 | } |
1658 | |
1659 | /** |
1660 | * INTERNAL |
1661 | */ |
1662 | @Override |
1663 | public int getMaxLengthInplaceLob() { |
1664 | throw DbException.throwInternalError(); |
1665 | } |
1666 | |
1667 | /** |
1668 | * INTERNAL |
1669 | */ |
1670 | @Override |
1671 | public String getLobCompressionAlgorithm(int type) { |
1672 | return null; |
1673 | } |
1674 | |
1675 | /** |
1676 | * INTERNAL |
1677 | */ |
1678 | @Override |
1679 | public Object getLobSyncObject() { |
1680 | return this; |
1681 | } |
1682 | |
1683 | /** |
1684 | * INTERNAL |
1685 | */ |
1686 | @Override |
1687 | public SmallLRUCache<String, String[]> getLobFileListCache() { |
1688 | return null; |
1689 | } |
1690 | |
1691 | /** |
1692 | * INTERNAL |
1693 | */ |
1694 | @Override |
1695 | public TempFileDeleter getTempFileDeleter() { |
1696 | return TempFileDeleter.getInstance(); |
1697 | } |
1698 | |
1699 | /** |
1700 | * INTERNAL |
1701 | */ |
1702 | @Override |
1703 | public LobStorageBackend getLobStorage() { |
1704 | return null; |
1705 | } |
1706 | |
1707 | /** |
1708 | * INTERNAL |
1709 | */ |
1710 | @Override |
1711 | public int readLob(long lobId, byte[] hmac, long offset, byte[] buff, |
1712 | int off, int length) { |
1713 | throw DbException.throwInternalError(); |
1714 | } |
1715 | |
1716 | @Override |
1717 | public JavaObjectSerializer getJavaObjectSerializer() { |
1718 | return null; |
1719 | } |
1720 | } |