dependencies

com.google.protobuf/protobuf-java		3.5.1
gloss		0.2.6
org.clojure/clojure		1.8.0
org.flatland/io		0.3.0

(this space intentionally left almost blank)

namespaces

protobuf.util
protobuf.schema
protobuf.codec
protobuf.core

protobuf.util

toc

(ns protobuf.util
  (:import
    (clojure.lang IDeref
                  ISeq
                  IPersistentMap
                  IPersistentSet
                  IPersistentCollection))
  (:refer-clojure :exclude [update]))

Turn an object into a fn if it is not already, by wrapping it in constantly.

(defn as-fn
  [x]
  (if (ifn? x) x, (constantly x)))

Update the given set using an existence map.

(defn into-set
  [set map]
  (if (map? map)
    (reduce (fn [set [k v]] ((if v conj disj) set k))
            set map)
    (into set map)))

Walk through clauses, a series of predicate/transform pairs. The first predicate that x satisfies has its transformation clause called on x. Predicates or transforms may be values (eg true or nil) rather than functions; these will be treated as functions that return that value. The last "pair" may be only a transform with no pred: in that case it is unconditionally used to transform x, if nothing previously matched. If no predicate matches, then x is returned unchanged.

(defn fix
  [x & clauses]
  (let [call #((as-fn %) x)]
    (first (or (seq (for [[pred & [transform :as exists?]] (partition-all 2 clauses)
                          :let [[pred transform] ;; handle odd number of clauses
                                (if exists? [pred transform] [true pred])]
                          :when (call pred)]
                      (call transform)))
               [x]))))

A "curried" version of fix, which sets the clauses once, yielding a function that calls fix with the specified first argument.

(defn to-fix
  [& clauses]
  (fn [x]
    (apply fix x clauses)))

A macro combining the features of fix and fixing, by using parentheses to group the additional arguments to each clause: (-> x (given string? read-string map? (dissoc :x :y :z) even? (/ 2)))

(defmacro given
  [x & clauses]
  (let [[clauses default] (if (even? (count clauses))
                            [clauses `identity]
                            [(butlast clauses) (last clauses)])]
    `(fix ~x ~@(for [[pred transform] (partition 2 clauses)
                     arg [pred `#(-> % ~transform)]]
                 arg)
          ~default)))

Create a clojure.lang.MapEntry from a and b. Equivalent to a cons cell. flatland.useful.experimental.unicode contains a shortcut to this, named ·.

(defmacro map-entry
  [a b]
  `(clojure.lang.MapEntry. ~a ~b))

Updates a value in a nested associative structure, where ks is a sequence of keys and f is a function that will take the old value and any supplied args and return the new value, and returns a new nested structure. If any levels do not exist, hash-maps will be created. This implementation was adapted from clojure.core, but the behavior is more correct if keys is empty and unchanged values are not re-assoc'd.

(defn update-in*
  [m keys f & args]
  (if-let [[k & ks] (seq keys)]
    (let [old (get m k)
          new (apply update-in* old ks f args)]
      (if (identical? old new)
        m
        (assoc m k new)))
     (apply f m args)))

Update a value for the given key in a map where f is a function that takes the previous value and the supplied args and returns the new value. Like update-in*, unchanged values are not re-assoc'd.

(defn update
  [m key f & args]
  (apply update-in* m [key] f args))

Create a new map from m by calling function f on each value to get a new value.

(defn map-vals
  [m f & args]
  (when m
    (into {}
          (for [[k v] m]
            (map-entry k (apply f v args))))))

(defprotocol Combiner
  (combine-onto [left right]
  "Merge two data structures by combining the contents. For maps, merge recursively by
  adjoining values with the same key. For collections, combine the right and left using
  into or conj. If the left value is a set and the right value is a map, the right value
  is assumed to be an existence map where the value determines whether the key is in the
  merged set. This makes sets unique from other collections because items can be deleted
  from them."))

(extend-protocol Combiner
  IPersistentMap
  (combine-onto [this other]
    (merge-with combine-onto this other))

  IPersistentSet
  (combine-onto [this other]
    (into-set this other))

  ISeq
  (combine-onto [this other]
    (concat this other))

  IPersistentCollection
  (combine-onto [this other]
    (into this other))

  Object
  (combine-onto [this other]
    other)

  nil
  (combine-onto [this other]
    other))

Merge two data structures by combining the contents. For maps, merge recursively by adjoining values with the same key. For collections, combine the right and left using into or conj. If the left value is a set and the right value is a map, the right value is assumed to be an existence map where the value determines whether the key is in the merged set. This makes sets unique from other collections because items can be deleted from them.

(defn combine
  [a b]
  (combine-onto a b))

(defn catbytes
  [& args]
  (let [out-buf (byte-array (loop [len 0, args (seq args)]
                              (if-let [[arg & args] args]
                                (recur (+ len (alength ^bytes arg)) args)
                                len)))]
    (loop [offset 0, args args]
      (if-let [[^bytes array & more] (seq args)]
        (let [size (alength array)]
          (System/arraycopy array 0
                            out-buf offset size)
          (recur (+ size offset) more))
        out-buf))))

Traverse all child types of the given schema, calling inner on each, then call outer on the result.

(defn walk
  [inner outer schema]
  (outer
   (case (:type schema)
     :struct           (update schema :fields map-vals inner)
     (:set :list :map) (-> schema
                           (given :values (update :values inner))
                           (given :keys   (update :keys   inner)))
     schema)))

Perform a depth-first, post-order traversal of all types within the given schema, replacing each and type with the result of calling f on it.

(defn postwalk
  [f schema]
  (walk (partial postwalk f) f schema))

Like postwalk, but do a pre-order traversal.

(defn prewalk
  [f schema]
  (walk (partial prewalk f) identity (f schema)))

Traverse the given schema, removing the given fields at any level.

(defn dissoc-fields
  [schema & fields]
  (prewalk (to-fix #(= :struct (:type %))
                   #(apply update % :fields dissoc fields))
           schema))

protobuf.schema

toc

(ns protobuf.schema
  (:require
    [clojure.string :as string]
    [protobuf.util :as util])
  (:import
    (protobuf Extensions
              PersistentProtocolBufferMap
              PersistentProtocolBufferMap$Def )
    (com.google.protobuf Descriptors$Descriptor
                         Descriptors$FieldDescriptor
                         Descriptors$FieldDescriptor$Type)))

(defn extension
  [ext ^Descriptors$FieldDescriptor field]
  (-> (.getOptions field)
      (.getExtension ext)
      (util/fix string? not-empty)))

(defn field-type
  [field]
  (condp instance? field
    Descriptors$FieldDescriptor
    (if (.isRepeated ^Descriptors$FieldDescriptor field)
      (condp extension field
        (Extensions/counter)    :counter
        (Extensions/succession) :succession
        (Extensions/map)        :map
        (Extensions/mapBy)      :map-by
        (Extensions/set)        :set
        :list)
      :basic)
    Descriptors$Descriptor
    :struct))

(defmulti field-schema
  (fn [field _ & _] (field-type field)))

(defn struct-schema
  [^Descriptors$Descriptor struct
   ^PersistentProtocolBufferMap$Def map-def
   & [parents]]
  (let [struct-name (.getFullName struct)]
    (into {:type :struct
           :name struct-name}
          (when (not-any? (partial = struct-name) parents)
            {:fields (into {}
                           (for [^Descriptors$FieldDescriptor field (.getFields struct)]
                             [(.intern map-def (.getName field))
                              (field-schema field map-def (conj parents struct-name))]))}))))

(defn basic-schema
  [^Descriptors$FieldDescriptor field
   ^PersistentProtocolBufferMap$Def map-def
   & [parents]]
  (let [java-type   (keyword (string/lower-case (.name (.getJavaType field))))
        meta-string (extension (Extensions/meta) field)]
    (merge (case java-type
             :message (struct-schema (.getMessageType field) map-def parents)
             :enum    {:type   :enum
                       :values (set (map #(.clojureEnumValue map-def %)
                                         (.. field getEnumType getValues)))}
             {:type java-type})
           (when (.hasDefaultValue field)
             {:default (.getDefaultValue field)})
           (when meta-string
             (read-string meta-string)))))

(defn subfield
  [^Descriptors$FieldDescriptor field field-name]
  (.findFieldByName (.getMessageType field) (name field-name)))

(defmethod field-schema :basic
  [field map-def & [parents]]
  (basic-schema field map-def parents))

(defmethod field-schema :list
  [field map-def & [parents]]
  {:type   :list
   :values (basic-schema field map-def parents)})

(defmethod field-schema :succession
  [field map-def & [parents]]
  (assoc (basic-schema field map-def parents)
    :succession true))

(defmethod field-schema :counter
  [field map-def & [parents]]
  (assoc (basic-schema field map-def parents)
    :counter true))

(defmethod field-schema :set
  [field map-def & [parents]]
  {:type   :set
   :values (field-schema (subfield field :item) map-def parents)})

(defmethod field-schema :map
  [field map-def & [parents]]
  {:type   :map
   :keys   (field-schema (subfield field :key) map-def parents)
   :values (field-schema (subfield field :val) map-def parents)})

(defmethod field-schema :map-by
  [field map-def & [parents]]
  (let [map-by (extension (Extensions/mapBy) field)]
    {:type   :map
     :keys   (field-schema (subfield field map-by) map-def parents)
     :values (basic-schema field map-def parents)}))

(defmethod field-schema :struct
  [field map-def & [parents]]
  (struct-schema field map-def parents))

protobuf.codec

toc

(ns protobuf.codec
  (:require
    [clojure.java.io :as io]
    ;; flatland.io extends Seqable so we can concat InputStream from
    ;; ByteBuffer sequences.
    [flatland.io.core]
    [gloss.core :as gloss]
    [gloss.core.formats :as gloss-formats]
    [gloss.io :as gloss-io]
    [gloss.core.protocols :as gloss-protocols]
    [protobuf.core :as protobuf]
    [protobuf.util :as util]))

(declare create)

(def ^{:private true} len-key :proto_length)
(def ^{:private true} reset-key :codec_reset)

(defn length-prefix
  [proto]
  (let [proto (protobuf/mapdef proto)
        min   (alength (protobuf/->bytes proto {len-key 0}))
        max   (alength (protobuf/->bytes proto {len-key Integer/MAX_VALUE}))]
    (letfn [(check [test msg]
              (when-not test
                (throw (Exception. (format "In %s: %s %s"
                                           (.getFullName proto) (name len-key) msg)))))]
      (check (pos? min)
             "field is required for repeated protobufs")
      (check (= min max)
             "must be of type fixed32 or fixed64"))
    (gloss/compile-frame (gloss/finite-frame max (create proto))
                         #(hash-map len-key %)
                         len-key)))

(defn create
  [proto & {:keys [validator repeated]}]
  (let [proto (protobuf/mapdef proto)]
    (-> (reify
          ;; Reader method
          gloss-protocols/Reader
          (read-bytes [this buf-seq]
            [true (protobuf/parse proto (io/input-stream buf-seq)) nil])
          ;; Writer method
          gloss-protocols/Writer
          (sizeof [this] nil)
          (write-bytes [this _ val]
            (when (and validator (not (validator val)))
              (throw (IllegalStateException. "Invalid value in #'protobuf.codec/create")))
            (gloss-formats/to-buf-seq
             (protobuf/->bytes
              (if (protobuf/map? val)
                val
                (protobuf/create proto val))))))
        (util/fix
          repeated
          #(gloss/repeated (gloss/finite-frame (length-prefix proto) %)
                           :prefix :none)))))

Aliases

Backwards-compatible alias for create

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protobuf-codec #'create)

Encoder for a codec.

(def 
  encode #'gloss-io/encode)

Decoder for a codec.

(def 
  decode #'gloss-io/decode)

XXX what is this used for? Delete, if not used; add docstring if it is

(defn codec-schema
  [proto]
  (util/dissoc-fields (protobuf/mapdef->schema proto)
                      len-key
                      reset-key))

protobuf.core

toc

(ns protobuf.core
  (:require
    [clojure.java.io :as io]
    [protobuf.schema :as protobuf-schema]
    [protobuf.util :as util])
  (:import
    (clojure.lang Reflector)
    (com.google.protobuf CodedInputStream
                         Descriptors$Descriptor
                         GeneratedMessage)
    (java.io InputStream OutputStream)
    (protobuf Extensions
              PersistentProtocolBufferMap
              PersistentProtocolBufferMap$Def
              PersistentProtocolBufferMap$Def$NamingStrategy))
  (:refer-clojure :exclude [map? read]))

Is the given object a PersistentProtocolBufferMap?

(defn map?
  [obj]
  (instance? PersistentProtocolBufferMap obj))

Is the given object a PersistentProtocolBufferMap$Def?

(defn mapdef?
  [obj]
  (instance? PersistentProtocolBufferMap$Def obj))

Create a protocol buffer map definition from a string or protobuf class.

(defn ^PersistentProtocolBufferMap$Def mapdef
  ([map-def]
     (if (or (mapdef? map-def) (nil? map-def))
       map-def
       (mapdef map-def {})))
  ([map-def opts]
     (when map-def
       (let [{:keys [^PersistentProtocolBufferMap$Def$NamingStrategy naming-strategy
                     size-limit]
              :or {naming-strategy PersistentProtocolBufferMap$Def/convertUnderscores
                   size-limit 67108864}} opts ;; 64MiB
             ^Descriptors$Descriptor descriptor
             (if (instance? Descriptors$Descriptor map-def)
               map-def
               (Reflector/invokeStaticMethod ^Class map-def "getDescriptor" (to-array nil)))]
         (PersistentProtocolBufferMap$Def/create descriptor naming-strategy size-limit)))))

Construct a protobuf of the given map-def.

(defn create
  ([^PersistentProtocolBufferMap$Def map-def]
     (PersistentProtocolBufferMap/construct map-def {}))
  ([^PersistentProtocolBufferMap$Def map-def m]
     (PersistentProtocolBufferMap/construct map-def m))
  ([^PersistentProtocolBufferMap$Def map-def k v & kvs]
     (PersistentProtocolBufferMap/construct map-def (apply array-map k v kvs))))

Return the schema for the given mapdef.

(defn mapdef->schema
  [& args]
  (let [^PersistentProtocolBufferMap$Def map-def (apply mapdef args)]
    (protobuf-schema/field-schema (.getMessageType map-def) map-def)))

Load a protobuf of the given map-def from a data source.

Supported data sources are either an array of bytes or an input stream.

(defmulti parse
  (fn [map-def data & _]
    (type data)))

(defmethod parse (Class/forName "[B")
  ([^PersistentProtocolBufferMap$Def map-def data]
     (when data
       (PersistentProtocolBufferMap/create map-def data)))
  ([^PersistentProtocolBufferMap$Def map-def data ^Integer offset ^Integer length]
     (when data
       (let [^CodedInputStream in (CodedInputStream/newInstance data offset length)]
         (PersistentProtocolBufferMap/parseFrom map-def in)))))

(defmethod parse InputStream
  [^PersistentProtocolBufferMap$Def map-def stream]
  (when stream
    (let [^CodedInputStream in (CodedInputStream/newInstance stream)]
      (PersistentProtocolBufferMap/parseFrom map-def in))))

Return the byte representation of the given protobuf.

(defn ^"[B" ->bytes
  ([^PersistentProtocolBufferMap p]
     (.toByteArray p))
  ([^PersistentProtocolBufferMap$Def map-def m]
     (->bytes (PersistentProtocolBufferMap/construct map-def m))))

Lazily read a sequence of length-delimited protobufs of the specified map-def from the given input stream.

(defn read
  [^PersistentProtocolBufferMap$Def map-def in]
  (lazy-seq
   (io!
    (let [^InputStream in (io/input-stream in)]
      (if-let [p (PersistentProtocolBufferMap/parseDelimitedFrom map-def in)]
        (cons p (read map-def in))
        (.close in))))))

Write the given protobufs to the given output stream, prefixing each with its length to delimit them.

(defn write
  [out & ps]
  (io!
   (let [^OutputStream out (io/output-stream out)]
     (doseq [^PersistentProtocolBufferMap p ps]
       (.writeDelimitedTo p out))
     (.flush out))))

(extend-protocol util/Combiner
  PersistentProtocolBufferMap
  (combine-onto [^PersistentProtocolBufferMap this other]
    (.append this other)))

A convenience alias for util/combine

(def 
  combine #'util/combine)

Get value at key ignoring extension fields.

TODO make this nil-safe? Or just delete it?

(defn get-raw
  [^PersistentProtocolBufferMap p key]
  (.getValAt p key false))

Aliases

Backwards-compatible alias for protobuf.core/map?

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protobuf? #'map?)

Backwards-compatible alias for mapdef?

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protodef? #'mapdef?)

Backwards-compatible alias for mapdef

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protodef #'mapdef)

Backwards-compatible alias for mapdef

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protobuf #'create)

Backwards-compatible alias for mapdef

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protobuf-schema #'mapdef->schema)

Backwards-compatible alias for parse

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protobuf-load #'parse)

Backwards-compatible alias for parse

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protobuf-load-stream #'parse)

Backwards-compatible alias for ->bytes

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protobuf-dump #'->bytes)

Backwards-compatible alias for read

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protobuf-seq #'read)

Backwards-compatible alias for write

(def ^{:doc 
       :deprecated "3.5.1-v1.0"}
  protobuf-write #'write)

Clojure Protocol Buffer Library

3.5.1-v1.0

dependencies

namespaces

protobuf.util

protobuf.schema

protobuf.codec

protobuf.core