auto merge of #8745 : brson/rust/metadata, r=cmr

This does two things: 1) stops compressing metadata, 2) stops copying the metadata section, instead holding a reference to the buffer returned by the LLVM section iterator.

Not compressing metadata requires something like 7x the storage space, but makes running tests about 9% faster. This has been a time improvement on all platforms I've tested, including windows. I considered leaving compression as an option but it doesn't seem to be worth the complexity since we don't currently have any use cases where we need to save that space.

In order to avoid copying the metadata section I had to hack up extra::ebml a bit to support unsafe buffers. We should probably move it into librustc so that it can evolve to support the compiler without worrying about having a crummy interface.

r? @graydon

Author: bors

configure

@@ -381,7 +381,8 @@ opt mingw-cross 0 "cross-compile for win32 using mingw"
 opt clang 0 "prefer clang to gcc for building the runtime"
 opt ccache 0 "invoke gcc/clang via ccache to reuse object files between builds"
 opt local-rust 0 "use an installed rustc rather than downloading a snapshot"
-opt pax-flags 0 "apply PaX flags to rustc binaries (required for GRSecurity/PaX-patched kernels)"
+opt pax-flags 0 "apply PaX flags to rustc binaries (required for GRSecurity/PaX-patched
+ kernels)"
 valopt prefix "/usr/local" "set installation prefix"
 valopt local-rust-root "/usr/local" "set prefix for local rust binary"
 valopt llvm-root "" "set LLVM root"

src/libextra/ebml.rs

@@ -12,6 +12,8 @@
 
 
 use std::str;
+use std::cast;
+use std::vec;
 
 // Simple Extensible Binary Markup Language (ebml) reader and writer on a
 // cursor model. See the specification here:
@@ -29,9 +31,42 @@ struct EbmlState {
     data_pos: uint,
 }
 
+#[deriving(Clone)]
+pub enum EbmlData {
+    SafeData(@~[u8]),
+    UnsafeData(*u8, uint)
+}
+
+impl EbmlData {
+    #[inline]
+    pub fn slice<'a>(&'a self, start: uint, end: uint) -> &'a [u8] {
+        match *self {
+            SafeData(@ref v) => v.slice(start, end),
+            UnsafeData(buf, len) => unsafe {
+                do vec::raw::buf_as_slice(buf, len) |s| {
+                    cast::transmute(s.slice(start, end))
+                }
+            }
+        }
+    }
+
+    #[inline]
+    pub fn as_slice<'a>(&'a self) -> &'a [u8] {
+        self.slice(0, self.len())
+    }
+
+    #[inline]
+    pub fn len(&self) -> uint {
+        match *self {
+            SafeData(@ref v) => v.len(),
+            UnsafeData(_, len) => len
+        }
+    }
+}
+
 #[deriving(Clone)]
 pub struct Doc {
-    data: @~[u8],
+    data: EbmlData,
     start: uint,
     end: uint,
 }
@@ -185,24 +220,28 @@ pub mod reader {
     }
 
     pub fn Doc(data: @~[u8]) -> Doc {
-        Doc { data: data, start: 0u, end: data.len() }
+        Doc { data: SafeData(data), start: 0u, end: data.len() }
+    }
+
+    pub fn unsafe_Doc(buf: *u8, len: uint) -> Doc {
+        Doc { data: UnsafeData(buf, len), start: 0u, end: len }
     }
 
-    pub fn doc_at(data: @~[u8], start: uint) -> TaggedDoc {
-        let elt_tag = vuint_at(*data, start);
-        let elt_size = vuint_at(*data, elt_tag.next);
+    pub fn doc_at(data: &EbmlData, start: uint) -> TaggedDoc {
+        let elt_tag = vuint_at(data.as_slice(), start);
+        let elt_size = vuint_at(data.as_slice(), elt_tag.next);
         let end = elt_size.next + elt_size.val;
         TaggedDoc {
             tag: elt_tag.val,
-            doc: Doc { data: data, start: elt_size.next, end: end }
+            doc: Doc { data: data.clone(), start: elt_size.next, end: end }
         }
     }
 
     pub fn maybe_get_doc(d: Doc, tg: uint) -> Option<Doc> {
         let mut pos = d.start;
         while pos < d.end {
-            let elt_tag = vuint_at(*d.data, pos);
-            let elt_size = vuint_at(*d.data, elt_tag.next);
+            let elt_tag = vuint_at(d.data.as_slice(), pos);
+            let elt_size = vuint_at(d.data.as_slice(), elt_tag.next);
             pos = elt_size.next + elt_size.val;
             if elt_tag.val == tg {
                 return Some(Doc { data: d.data, start: elt_size.next,
@@ -225,8 +264,8 @@ pub mod reader {
     pub fn docs(d: Doc, it: &fn(uint, Doc) -> bool) -> bool {
         let mut pos = d.start;
         while pos < d.end {
-            let elt_tag = vuint_at(*d.data, pos);
-            let elt_size = vuint_at(*d.data, elt_tag.next);
+            let elt_tag = vuint_at(d.data.as_slice(), pos);
+            let elt_size = vuint_at(d.data.as_slice(), elt_tag.next);
             pos = elt_size.next + elt_size.val;
             let doc = Doc { data: d.data, start: elt_size.next, end: pos };
             if !it(elt_tag.val, doc) {
@@ -239,8 +278,8 @@ pub mod reader {
     pub fn tagged_docs(d: Doc, tg: uint, it: &fn(Doc) -> bool) -> bool {
         let mut pos = d.start;
         while pos < d.end {
-            let elt_tag = vuint_at(*d.data, pos);
-            let elt_size = vuint_at(*d.data, elt_tag.next);
+            let elt_tag = vuint_at(d.data.as_slice(), pos);
+            let elt_size = vuint_at(d.data.as_slice(), elt_tag.next);
             pos = elt_size.next + elt_size.val;
             if elt_tag.val == tg {
                 let doc = Doc { data: d.data, start: elt_size.next,
@@ -260,22 +299,22 @@ pub mod reader {
 
     pub fn doc_as_u8(d: Doc) -> u8 {
         assert_eq!(d.end, d.start + 1u);
-        (*d.data)[d.start]
+        d.data.as_slice()[d.start]
     }
 
     pub fn doc_as_u16(d: Doc) -> u16 {
         assert_eq!(d.end, d.start + 2u);
-        io::u64_from_be_bytes(*d.data, d.start, 2u) as u16
+        io::u64_from_be_bytes(d.data.as_slice(), d.start, 2u) as u16
     }
 
     pub fn doc_as_u32(d: Doc) -> u32 {
         assert_eq!(d.end, d.start + 4u);
-        io::u64_from_be_bytes(*d.data, d.start, 4u) as u32
+        io::u64_from_be_bytes(d.data.as_slice(), d.start, 4u) as u32
     }
 
     pub fn doc_as_u64(d: Doc) -> u64 {
         assert_eq!(d.end, d.start + 8u);
-        io::u64_from_be_bytes(*d.data, d.start, 8u)
+        io::u64_from_be_bytes(d.data.as_slice(), d.start, 8u)
     }
 
     pub fn doc_as_i8(d: Doc) -> i8 { doc_as_u8(d) as i8 }
@@ -298,8 +337,7 @@ pub mod reader {
     impl Decoder {
         fn _check_label(&mut self, lbl: &str) {
             if self.pos < self.parent.end {
-                let TaggedDoc { tag: r_tag, doc: r_doc } =
-                    doc_at(self.parent.data, self.pos);
+                let TaggedDoc { tag: r_tag, doc: r_doc } = doc_at(&self.parent.data, self.pos);
 
                 if r_tag == (EsLabel as uint) {
                     self.pos = r_doc.end;
@@ -316,8 +354,7 @@ pub mod reader {
             if self.pos >= self.parent.end {
                 fail!("no more documents in current node!");
             }
-            let TaggedDoc { tag: r_tag, doc: r_doc } =
-                doc_at(self.parent.data, self.pos);
+            let TaggedDoc { tag: r_tag, doc: r_doc } = doc_at(&self.parent.data, self.pos);
             debug!("self.parent=%?-%? self.pos=%? r_tag=%? r_doc=%?-%?",
                    self.parent.start,
                    self.parent.end,

src/librustc/driver/driver.rs

@@ -763,7 +763,7 @@ pub fn build_session_options(binary: @str,
         parse_only: parse_only,
         no_trans: no_trans,
         debugging_opts: debugging_opts,
-        android_cross_path: android_cross_path
+        android_cross_path: android_cross_path,
     };
     return sopts;
 }

src/librustc/metadata/creader.rs

@@ -15,6 +15,7 @@ use metadata::cstore;
 use metadata::decoder;
 use metadata::filesearch::FileSearch;
 use metadata::loader;
+use metadata::loader::MetadataSection;
 
 use std::hashmap::HashMap;
 use syntax::ast;
@@ -308,7 +309,7 @@ fn resolve_crate(e: @mut Env,
 }
 
 // Go through the crate metadata and load any crates that it references
-fn resolve_crate_deps(e: @mut Env, cdata: @~[u8]) -> cstore::cnum_map {
+fn resolve_crate_deps(e: @mut Env, cdata: MetadataSection) -> cstore::cnum_map {
     debug!("resolving deps of external crate");
     // The map from crate numbers in the crate we're resolving to local crate
     // numbers

src/librustc/metadata/csearch.rs

@@ -188,7 +188,7 @@ pub fn get_field_type(tcx: ty::ctxt, class_id: ast::def_id,
                       def: ast::def_id) -> ty::ty_param_bounds_and_ty {
     let cstore = tcx.cstore;
     let cdata = cstore::get_crate_data(cstore, class_id.crate);
-    let all_items = reader::get_doc(reader::Doc(cdata.data), tag_items);
+    let all_items = reader::get_doc(decoder::section_to_ebml_doc(cdata.data), tag_items);
     debug!("Looking up %?", class_id);
     let class_doc = expect(tcx.diag,
                            decoder::maybe_find_item(class_id.node, all_items),

src/librustc/metadata/cstore.rs

@@ -15,6 +15,7 @@
 
 use metadata::cstore;
 use metadata::decoder;
+use metadata::loader::MetadataSection;
 
 use std::hashmap::HashMap;
 use extra;
@@ -29,7 +30,7 @@ pub type cnum_map = @mut HashMap<ast::CrateNum, ast::CrateNum>;
 
 pub struct crate_metadata {
     name: @str,
-    data: @~[u8],
+    data: MetadataSection,
     cnum_map: cnum_map,
     cnum: ast::CrateNum
 }