Бинарный формат PSON

  # Codes of data types in PSON
  # RU: Коды типов данных в PSON
  PT_Int   = 0
  PT_Str   = 1
  PT_Bool  = 2
  PT_Time  = 3
  PT_Array = 4
  PT_Hash  = 5
  PT_Sym   = 6
  PT_Real  = 7
  # 8..14 - reserved for other types
  PT_Nil   = 15
  PT_Negative = 16

  # Encode data type and size to PSON type and count of size in bytes (1..8)-1
  # RU: Кодирует тип данных и размер в тип PSON и число байт размера
  def self.encode_pson_type(basetype, int)
    count = 0
    neg = 0
    if int<0
      neg = PT_Negative
      int = -int
    end
    while (int>0) and (count<8)
      int = (int >> 8)
      count +=1
    end
    if count >= 8
      puts '[encode_pan_type] Too big int='+int.to_s
      count = 7
    end
    [basetype ^ neg ^ (count << 5), count, (neg>0)]
  end

  # Decode PSON type to data type and count of size in bytes (1..8)-1
  # RU: Раскодирует тип PSON в тип данных и число байт размера
  def self.decode_pson_type(type)
    basetype = type & 0xF
    negative = ((type & PT_Negative)>0)
    count = (type >> 5)
    [basetype, count, negative]
  end

  # Convert ruby object to PSON (Pandora Simple Object Notation)
  # RU: Конвертирует объект руби в PSON
  # sort_mode: nil or false - don't sort, 1 - sort array, 2 - sort hash
  # 3 or true - sort arrays and hashes 
  def self.rubyobj_to_pson(rubyobj, sort_mode=nil)
    type = PT_Nil
    count = 0
    neg = false
    data = AsciiString.new
    elem_size = nil
    case rubyobj
      when String
        data << AsciiString.new(rubyobj)
        elem_size = data.bytesize
        type, count, neg = encode_pson_type(PT_Str, elem_size)
      when Symbol
        data << AsciiString.new(rubyobj.to_s)
        elem_size = data.bytesize
        type, count, neg = encode_pson_type(PT_Sym, elem_size)
      when Integer
        type, count, neg = encode_pson_type(PT_Int, rubyobj)
        rubyobj = -rubyobj if neg
        data << PandoraUtils.bigint_to_bytes(rubyobj)
      when Time
        rubyobj = rubyobj.to_i
        type, count, neg = encode_pson_type(PT_Time, rubyobj)
        rubyobj = -rubyobj if neg
        data << PandoraUtils.bigint_to_bytes(rubyobj)
      when TrueClass, FalseClass
        type = PT_Bool
        type = type ^ PT_Negative if not rubyobj
      when Float
        data << [rubyobj].pack('D')
        elem_size = data.bytesize
        type, count, neg = encode_pson_type(PT_Real, elem_size)
      when Array
        if (sort_mode and ((not sort_mode.is_a?(Integer)) or ((sort_mode & 1)>0)))
          rubyobj = self.sort_complex_array(rubyobj)
        end
        rubyobj.each do |a|
          data << rubyobj_to_pson(a, sort_mode)
        end
        elem_size = rubyobj.size
        type, count, neg = encode_pson_type(PT_Array, elem_size)
      when Hash
        if (sort_mode and ((not sort_mode.is_a?(Integer)) or ((sort_mode & 2)>0)))
          #rubyobj = rubyobj.sort_by {|k,v| k.to_s}
          rubyobj = self.sort_complex_hash(rubyobj)
        end
        elem_size = 0
        rubyobj.each do |a|
          data << rubyobj_to_pson(a[0], sort_mode) << rubyobj_to_pson(a[1], sort_mode)
          elem_size += 1
        end
        type, count, neg = encode_pson_type(PT_Hash, elem_size)
      when NilClass
        type = PT_Nil
      else
        puts 'Error! rubyobj_to_pson: illegal ruby class ['+rubyobj.class.name+']'
    end
    res = AsciiString.new
    res << [type].pack('C')
    if (data.is_a? String) and (count>0)
      data = AsciiString.new(data)
      if elem_size
        if (elem_size == data.bytesize) or (rubyobj.is_a? Array) or (rubyobj.is_a? Hash)
          res << PandoraUtils.fill_zeros_from_left( 
            PandoraUtils.bigint_to_bytes(elem_size), count) + data
        else
          puts 'Error! rubyobj_to_pson: elem_size<>data_size: '+elem_size.inspect+'<>'
            +data.bytesize.inspect + ' data='+data.inspect + ' rubyobj='+rubyobj.inspect
        end
      elsif data.bytesize>0
        res << PandoraUtils.fill_zeros_from_left(data, count)
      end
    end
    res = AsciiString.new(res)
  end

  # Convert PSON to ruby object
  # RU: Конвертирует PSON в объект руби
  def self.pson_to_rubyobj(data)
    data = AsciiString.new(data)
    val = nil
    len = 0
    if data.bytesize>0
      type = data[0].ord
      len = 1
      basetype, count, neg = decode_pson_type(type)
      if data.bytesize >= len+count
        elem_size = 0
        elem_size = PandoraUtils.bytes_to_int(data[len, count]) if count>0
        case basetype
          when PT_Int
            val = elem_size
            val = -val if neg
          when PT_Time
            val = elem_size
            val = -val if neg
            val = Time.at(val)
          when PT_Bool
            if count>0
              val = (elem_size != 0)
            else
              val = (not neg)
            end
          when PT_Str, PT_Sym, PT_Real
            pos = len+count
            if pos+elem_size>data.bytesize
              elem_size = data.bytesize-pos
            end
            val = AsciiString.new(data[pos, elem_size])
            count += elem_size
            if basetype == PT_Sym
              val = val.to_sym
            elsif basetype == PT_Real
              val = val.unpack('D')[0]
            end
          when PT_Array, PT_Hash
            val = Array.new
            elem_size *= 2 if basetype == PT_Hash
            while (data.bytesize-1-count>0) and (elem_size>0)
              elem_size -= 1
              aval, alen = pson_to_rubyobj(data[len+count..-1])
              val << aval
              count += alen
            end
            val = Hash[*val] if basetype == PT_Hash
          when PT_Nil
            val = nil
          else
            puts 'pson_to_rubyobj: illegal pson type '+basetype.inspect
        end
        len += count
      else
        len = data.bytesize
      end
    end
    [val, len]
  end

  # Value is empty?
  # RU: Значение пустое?
  def self.value_is_empty?(val)
    res = (val==nil) or (val.is_a? String and (val=='')) 
      or (val.is_a? Integer and (val==0)) or (val.is_a? Time and (val.to_i==0)) 
      or (val.is_a? Array and (val==[])) or (val.is_a? Hash and (val=={}))
    res
  end

  # Pack PanObject fields to Name-PSON binary format
  # RU: Пакует поля панобъекта в бинарный формат Name-PSON
  def self.hash_to_namepson(fldvalues, pack_empty=false, sort_mode=2)
    #bytes = ''
    #bytes.force_encoding('ASCII-8BIT')
    bytes = AsciiString.new
    fldvalues = fldvalues.sort_by {|k,v| k.to_s } if sort_mode
    fldvalues.each do |nam, val|
      if pack_empty or (not value_is_empty?(val))
        nam = nam.to_s
        nsize = nam.bytesize
        nsize = 255 if nsize>255
        bytes << [nsize].pack('C') + nam[0, nsize]
        pson_elem = rubyobj_to_pson(val, sort_mode)
        bytes << pson_elem
      end
    end
    bytes = AsciiString.new(bytes)
  end

  # Convert Name-PSON block to PanObject fields
  # RU: Преобразует Name-PSON блок в поля панобъекта
  def self.namepson_to_hash(pson)
    hash = {}
    while pson and (pson.bytesize>1)
      flen = pson[0].ord
      fname = pson[1, flen]
      if (flen>0) and fname and (fname.bytesize>0)
        val = nil
        if pson.bytesize-flen>1
          pson = pson[1+flen..-1]  # drop getted name
          val, len = pson_to_rubyobj(pson)
          pson = pson[len..-1]     # drop getted value
        else
          pson = nil
        end
        hash[fname] = val
      else
        pson = nil
        hash = nil if hash == {}
      end
    end
    hash
  end

# Codes of data types in PSON
# RU: Коды типов данных в PSON
PT_Int   = 0
PT_Str   = 1
PT_Bool  = 2
PT_Time  = 3
PT_Array = 4
PT_Hash  = 5
PT_Sym   = 6
PT_Real  = 7
# 8..14 - reserved for other types
PT_Nil   = 15
PT_Negative = 16

# Encode data type and size to PSON kind and count of size in bytes (1..8)-1
# RU: Кодирует тип данных и размер в тип PSON и число байт размера
def encode_pson_kind(basekind, size):
  count = 0
  neg = 0
  if size<0:
    neg = PT_Negative
    size = -size
  while (size>0) and (count<8):
    size = (size >> 8)
    count +=1
  if count >= 8:
    print('[encode_pan_kind] Too big int='+size.to_s)
    count = 7
  return [basekind ^ neg ^ (count << 5), count, (neg>0)]

# Decode PSON kind to data kind and count of size in bytes (1..8)-1
# RU: Раскодирует тип PSON в тип данных и число байт размера
def decode_pson_kind(kind):
  basekind = kind & 0xF
  negative = ((kind & PT_Negative)>0)
  count = (kind >> 5)
  return [basekind, count, negative]

# Convert python object to PSON (Pandora simple object notation)
# RU: Конвертирует объект питон в PSON
def pythonobj_to_pson(pythonobj):
  kind = PT_Nil
  count = 0
  data = '' #!!!data = AsciiString.new
  elem_size = None
  if isinstance(pythonobj, str):
    data += pythonobj #!!!AsciiString.new(pythonobj)
    elem_size = len(data) #!!!data.bytesize
    kind, count, neg = encode_pson_kind(PT_Str, elem_size)
  elif isinstance(pythonobj, bool):
    kind = PT_Bool
    print('Boool1 kind='+str(kind))
    if not pythonobj: kind = kind ^ PT_Negative
    print('Boool2 kind='+str(kind))
  elif isinstance(pythonobj, int):
    kind, count, neg = encode_pson_kind(PT_Int, pythonobj)
    if neg: pythonobj = -pythonobj
    data += bigint_to_bytes(pythonobj, 8)
  #!!!elif isinstance(pythonobj, Symbol):
  #  data << AsciiString.new(pythonobj.to_s)
  #  elem_size = data.bytesize
  #  kind, count, neg = encode_pson_kind(PT_Sym, elem_size)
  elif isinstance(pythonobj, datetime.datetime):
    pythonobj = int(pythonobj)
    kind, count, neg = encode_pson_kind(PT_Time, pythonobj)
    if neg: pythonobj = -pythonobj
    data << PandoraUtils.bigint_to_bytes(pythonobj)
  elif isinstance(pythonobj, float):
    data += struct.pack('d', pythonobj)
    elem_size = len(data)
    kind, count, neg = encode_pson_kind(PT_Real, elem_size)
  elif isinstance(pythonobj, (list, tuple)):
    for a in pythonobj:
      data += pythonobj_to_pson(a)
    elem_size = len(pythonobj)
    kind, count, neg = encode_pson_kind(PT_Array, elem_size)
  elif isinstance(pythonobj, dict):
    #!!!pythonobj = pythonobj.sort_by {|k,v| k.to_s}
    elem_size = 0
    for key in pythonobj:
      data += pythonobj_to_pson(key) + pythonobj_to_pson(pythonobj.get(key, 0))
      elem_size += 1
    kind, count, neg = encode_pson_kind(PT_Hash, elem_size)
  elif pythonobj is None:
    kind = PT_Nil
  else:
    print('Error! pythonobj_to_pson: illegal ruby class ['+pythonobj+']')
  res = ''   #res = AsciiString.new
  res += struct.pack('!B', kind)  #res << [kind].pack('C')
  if isinstance(data, str) and (count>0):
    #!!!data = AsciiString.new(data)
    if elem_size:
      if (elem_size==len(data)) or isinstance(pythonobj, (list,dict,tuple)):
        #!!!res += PandoraUtils.fill_zeros_from_left( 
        #  PandoraUtils.bigint_to_bytes(elem_size), count) + data
        res += fill_zeros_from_left(bigint_to_bytes(elem_size), count) + data
      else:
        print('Error! pythonobj_to_pson: elem_size<>data_size: '+elem_size.inspect+'<>'
          +data.bytesize.inspect + ' data='+data.inspect + ' pythonobj='+pythonobj.inspect)
    elif len(data)>0:
      #!!!res << PandoraUtils.fill_zeros_from_left(data, count)
      res += data[:count]
  return res #AsciiString.new(res)

# Convert PSON to python object
# RU: Конвертирует PSON в объект питон
def pson_to_pythonobj(data):
  val = None
  size = 0
  if len(data)>0:
    kind = ord(data[0])
    size = 1
    basekind, count, neg = decode_pson_kind(kind)
    if (len(data) >= size+count):
      elem_size = 0
      if count>0: elem_size = bytes_to_int(data[size:size+count])
      if basekind==PT_Int:
        val = elem_size
        if neg: val = -val
      elif basekind==PT_Time:
        val = elem_size
        if neg: val = -val
        val = datetime.datetime(val)  #Time.at(val)
      elif basekind==PT_Bool:
        if count>0:
          val = (elem_size != 0)
        else:
          val = (not neg)
      elif (basekind==PT_Str) or (basekind==PT_Sym) or (basekind==PT_Real):
        pos = size+count
        if pos+elem_size>len(data):
          elem_size = len(data)-pos
        val = data[pos: pos+elem_size]
        count += elem_size
        if basekind == PT_Sym:
          val = val.to_sym
        elif basekind == PT_Real:
          unpacked = struct.unpack('d', val)
          val = unpacked[0]
          print('RT_REAL val='+str(val))
      elif (basekind==PT_Array) or (basekind==PT_Hash):
        val = []
        if basekind == PT_Hash: elem_size *= 2
        while (len(data)-1-count>0) and (elem_size>0):
          elem_size -= 1
          aval, alen = pson_to_pythonobj(data[size+count:])
          val.append(aval)
          count += alen
        if basekind == PT_Hash:
          dic = {}
          for i in range(len(val)/2): dic[val[i*2]] = val[i*2+1]
          val = dic
          print(str(val))
      elif (basekind==PT_Nil):
        val = None
      else:
        print('pson_to_pythonobj: illegal pson kind '+basekind.inspect)
      size += count
    else:
      size = data.bytesize
  return [val, size]

# Value is empty?
# RU: Значение пустое?
def is_value_empty(val):
  res = ((val is None) or (isinstance(val, str) and (len(val)==0)) 
    or (isinstance(val, int) and (val==0)) 
    or (isinstance(val, list) and (val==[])) or (isinstance(val, dict) and (val=={})))
    #or (val==Time and (val.to_i==0)) 
  return res

# Pack PanObject fields to Name-PSON binary format
# RU: Пакует поля панобъекта в бинарный формат Name-PSON
def hash_to_namepson(fldvalues, pack_empty=False):
  buf = ''
  #fldvalues = fldvalues.sort_by_key()  #!!!sort_by {|k,v| k.to_s } # sort by key
  for nam in fldvalues:
    val = fldvalues.get(nam, 0)
    if pack_empty or (not is_value_empty(val)):
      nam = str(nam)
      nsize = len(nam)
      if nsize>255: nsize = 255
      buf += struct.pack('B', nsize) + nam[0: nsize]
      pson_elem = pythonobj_to_pson(val)
      buf += pson_elem
  return buf


# Convert Name-PSON block to PanObject fields
# RU: Преобразует Name-PSON блок в поля панобъекта
def namepson_to_hash(pson):
  dic = {}
  while pson and (len(pson)>1):
    flen = ord(pson[0])
    fname = pson[1: 1+flen]
    if (flen>0) and fname and (len(fname)>0):
      val = None
      if len(pson)-flen > 1:
        pson = pson[1+flen:]  #!!! pson[1+flen..-1] # drop getted name
        val, size = pson_to_pythonobj(pson)
        pson = pson[size:]  #!!!pson[len..-1]   # drop getted value
      else:
        pson = None
      dic[fname] = val
    else:
      pson = None
      if dic == {}: dic = None
  return dic