first commit

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,5 @@
+#VDK
+
+Base Pack thx nareix created JOY4
+
+A set of libraries for building streaming services.

av/av.go

@@ -0,0 +1,316 @@
+
+// Package av defines basic interfaces and data structures of container demux/mux and audio encode/decode.
+package av
+
+import (
+	"fmt"
+	"time"
+)
+
+// Audio sample format.
+type SampleFormat uint8
+
+const (
+	U8 = SampleFormat(iota + 1) // 8-bit unsigned integer
+	S16 // signed 16-bit integer
+	S32 // signed 32-bit integer
+	FLT // 32-bit float
+	DBL // 64-bit float
+	U8P // 8-bit unsigned integer in planar
+	S16P // signed 16-bit integer in planar
+	S32P // signed 32-bit integer in planar
+	FLTP // 32-bit float in planar
+	DBLP // 64-bit float in planar
+	U32 // unsigned 32-bit integer
+)
+
+func (self SampleFormat) BytesPerSample() int {
+	switch self {
+	case U8, U8P:
+		return 1
+	case S16, S16P:
+		return 2
+	case FLT, FLTP, S32, S32P, U32:
+		return 4
+	case DBL, DBLP:
+		return 8
+	default:
+		return 0
+	}
+}
+
+func (self SampleFormat) String() string {
+	switch self {
+	case U8:
+		return "U8"
+	case S16:
+		return "S16"
+	case S32:
+		return "S32"
+	case FLT:
+		return "FLT"
+	case DBL:
+		return "DBL"
+	case U8P:
+		return "U8P"
+	case S16P:
+		return "S16P"
+	case FLTP:
+		return "FLTP"
+	case DBLP:
+		return "DBLP"
+	case U32:
+		return "U32"
+	default:
+		return "?"
+	}
+}
+
+// Check if this sample format is in planar.
+func (self SampleFormat) IsPlanar() bool {
+	switch self {
+	case S16P, S32P, FLTP, DBLP:
+		return true
+	default:
+		return false
+	}
+}
+
+// Audio channel layout.
+type ChannelLayout uint16
+
+func (self ChannelLayout) String() string {
+	return fmt.Sprintf("%dch", self.Count())
+}
+
+const (
+	CH_FRONT_CENTER = ChannelLayout(1 << iota)
+	CH_FRONT_LEFT
+	CH_FRONT_RIGHT
+	CH_BACK_CENTER
+	CH_BACK_LEFT
+	CH_BACK_RIGHT
+	CH_SIDE_LEFT
+	CH_SIDE_RIGHT
+	CH_LOW_FREQ
+	CH_NR
+
+	CH_MONO     = ChannelLayout(CH_FRONT_CENTER)
+	CH_STEREO   = ChannelLayout(CH_FRONT_LEFT | CH_FRONT_RIGHT)
+	CH_2_1      = ChannelLayout(CH_STEREO | CH_BACK_CENTER)
+	CH_2POINT1  = ChannelLayout(CH_STEREO | CH_LOW_FREQ)
+	CH_SURROUND = ChannelLayout(CH_STEREO | CH_FRONT_CENTER)
+	CH_3POINT1  = ChannelLayout(CH_SURROUND | CH_LOW_FREQ)
+	// TODO: add all channel_layout in ffmpeg
+)
+
+func (self ChannelLayout) Count() (n int) {
+	for self != 0 {
+		n++
+		self = (self - 1) & self
+	}
+	return
+}
+
+// Video/Audio codec type. can be H264/AAC/SPEEX/...
+type CodecType uint32
+
+var (
+	H264 = MakeVideoCodecType(avCodecTypeMagic + 1)
+	AAC       = MakeAudioCodecType(avCodecTypeMagic + 1)
+	PCM_MULAW = MakeAudioCodecType(avCodecTypeMagic + 2)
+	PCM_ALAW  = MakeAudioCodecType(avCodecTypeMagic + 3)
+	SPEEX = MakeAudioCodecType(avCodecTypeMagic + 4)
+	NELLYMOSER = MakeAudioCodecType(avCodecTypeMagic + 5)
+)
+
+const codecTypeAudioBit = 0x1
+const codecTypeOtherBits = 1
+
+func (self CodecType) String() string {
+	switch self {
+	case H264:
+		return "H264"
+	case AAC:
+		return "AAC"
+	case PCM_MULAW:
+		return "PCM_MULAW"
+	case PCM_ALAW:
+		return "PCM_ALAW"
+	case SPEEX:
+		return "SPEEX"
+	case NELLYMOSER:
+		return "NELLYMOSER"
+	}
+	return ""
+}
+
+func (self CodecType) IsAudio() bool {
+	return self&codecTypeAudioBit != 0
+}
+
+func (self CodecType) IsVideo() bool {
+	return self&codecTypeAudioBit == 0
+}
+
+// Make a new audio codec type.
+func MakeAudioCodecType(base uint32) (c CodecType) {
+	c = CodecType(base)<<codecTypeOtherBits | CodecType(codecTypeAudioBit)
+	return
+}
+
+// Make a new video codec type.
+func MakeVideoCodecType(base uint32) (c CodecType) {
+	c = CodecType(base) << codecTypeOtherBits
+	return
+}
+
+const avCodecTypeMagic = 233333
+
+// CodecData is some important bytes for initializing audio/video decoder,
+// can be converted to VideoCodecData or AudioCodecData using:
+//
+//     codecdata.(AudioCodecData) or codecdata.(VideoCodecData)
+// 
+// for H264, CodecData is AVCDecoderConfigure bytes, includes SPS/PPS.
+type CodecData interface {
+	Type() CodecType // Video/Audio codec type
+}
+
+type VideoCodecData interface {
+	CodecData
+	Width() int // Video width
+	Height() int // Video height
+}
+
+type AudioCodecData interface {
+	CodecData
+	SampleFormat() SampleFormat // audio sample format
+	SampleRate() int // audio sample rate
+	ChannelLayout() ChannelLayout // audio channel layout
+	PacketDuration([]byte) (time.Duration, error) // get audio compressed packet duration
+}
+
+type PacketWriter interface {
+	WritePacket(Packet) error
+}
+
+type PacketReader interface {
+	ReadPacket() (Packet,error)
+}
+
+// Muxer describes the steps of writing compressed audio/video packets into container formats like MP4/FLV/MPEG-TS.
+// 
+// Container formats, rtmp.Conn, and transcode.Muxer implements Muxer interface.
+type Muxer interface {
+	WriteHeader([]CodecData) error // write the file header
+	PacketWriter // write compressed audio/video packets
+	WriteTrailer() error // finish writing file, this func can be called only once
+}
+
+// Muxer with Close() method
+type MuxCloser interface {
+	Muxer
+	Close() error
+}
+
+// Demuxer can read compressed audio/video packets from container formats like MP4/FLV/MPEG-TS.
+type Demuxer interface {
+	PacketReader // read compressed audio/video packets
+	Streams() ([]CodecData, error) // reads the file header, contains video/audio meta infomations
+}
+
+// Demuxer with Close() method
+type DemuxCloser interface {
+	Demuxer
+	Close() error
+}
+
+// Packet stores compressed audio/video data.
+type Packet struct {
+	IsKeyFrame      bool // video packet is key frame
+	Idx             int8 // stream index in container format
+	CompositionTime time.Duration // packet presentation time minus decode time for H264 B-Frame
+	Time time.Duration // packet decode time
+	Data            []byte // packet data
+}
+
+// Raw audio frame.
+type AudioFrame struct {
+	SampleFormat  SampleFormat // audio sample format, e.g: S16,FLTP,...
+	ChannelLayout ChannelLayout // audio channel layout, e.g: CH_MONO,CH_STEREO,...
+	SampleCount   int // sample count in this frame
+	SampleRate    int // sample rate
+	Data          [][]byte // data array for planar format len(Data) > 1
+}
+
+func (self AudioFrame) Duration() time.Duration {
+	return time.Second * time.Duration(self.SampleCount) / time.Duration(self.SampleRate)
+}
+
+// Check this audio frame has same format as other audio frame.
+func (self AudioFrame) HasSameFormat(other AudioFrame) bool {
+	if self.SampleRate != other.SampleRate {
+		return false
+	}
+	if self.ChannelLayout != other.ChannelLayout {
+		return false
+	}
+	if self.SampleFormat != other.SampleFormat {
+		return false
+	}
+	return true
+}
+
+// Split sample audio sample from this frame.
+func (self AudioFrame) Slice(start int, end int) (out AudioFrame) {
+	if start > end {
+		panic(fmt.Sprintf("av: AudioFrame split failed start=%d end=%d invalid", start, end))
+	}
+	out = self
+	out.Data = append([][]byte(nil), out.Data...)
+	out.SampleCount = end - start
+	size := self.SampleFormat.BytesPerSample()
+	for i := range out.Data {
+		out.Data[i] = out.Data[i][start*size : end*size]
+	}
+	return
+}
+
+// Concat two audio frames.
+func (self AudioFrame) Concat(in AudioFrame) (out AudioFrame) {
+	out = self
+	out.Data = append([][]byte(nil), out.Data...)
+	out.SampleCount += in.SampleCount
+	for i := range out.Data {
+		out.Data[i] = append(out.Data[i], in.Data[i]...)
+	}
+	return
+}
+
+// AudioEncoder can encode raw audio frame into compressed audio packets.
+// cgo/ffmpeg inplements AudioEncoder, using ffmpeg.NewAudioEncoder to create it.
+type AudioEncoder interface {
+	CodecData() (AudioCodecData, error) // encoder's codec data can put into container
+	Encode(AudioFrame) ([][]byte, error) // encode raw audio frame into compressed pakcet(s)
+	Close() // close encoder, free cgo contexts
+	SetSampleRate(int) (error) // set encoder sample rate
+	SetChannelLayout(ChannelLayout) (error) // set encoder channel layout
+	SetSampleFormat(SampleFormat) (error) // set encoder sample format
+	SetBitrate(int) (error) // set encoder bitrate
+	SetOption(string,interface{}) (error) // encoder setopt, in ffmpeg is av_opt_set_dict()
+	GetOption(string,interface{}) (error) // encoder getopt
+}
+
+// AudioDecoder can decode compressed audio packets into raw audio frame.
+// use ffmpeg.NewAudioDecoder to create it.
+type AudioDecoder interface {
+	Decode([]byte) (bool, AudioFrame, error) // decode one compressed audio packet
+	Close() // close decode, free cgo contexts
+}
+
+// AudioResampler can convert raw audio frames in different sample rate/format/channel layout.
+type AudioResampler interface {
+	Resample(AudioFrame) (AudioFrame, error) // convert raw audio frames
+}
+

av/avconv/avconv.go

@@ -0,0 +1,255 @@
+package avconv
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/av/avutil"
+	"github.com/deepch/vdk/av/pktque"
+	"github.com/deepch/vdk/av/transcode"
+)
+
+var Debug bool
+
+type Option struct {
+	Transcode bool
+	Args      []string
+}
+
+type Options struct {
+	OutputCodecTypes []av.CodecType
+}
+
+type Demuxer struct {
+	transdemux *transcode.Demuxer
+	streams    []av.CodecData
+	Options
+	Demuxer av.Demuxer
+}
+
+func (self *Demuxer) Close() (err error) {
+	if self.transdemux != nil {
+		return self.transdemux.Close()
+	}
+	return
+}
+
+func (self *Demuxer) Streams() (streams []av.CodecData, err error) {
+	if err = self.prepare(); err != nil {
+		return
+	}
+	streams = self.streams
+	return
+}
+
+func (self *Demuxer) ReadPacket() (pkt av.Packet, err error) {
+	if err = self.prepare(); err != nil {
+		return
+	}
+	return self.transdemux.ReadPacket()
+}
+
+func (self *Demuxer) prepare() (err error) {
+	if self.transdemux != nil {
+		return
+	}
+
+	/*
+		var streams []av.CodecData
+		if streams, err = self.Demuxer.Streams(); err != nil {
+			return
+		}
+	*/
+
+	supports := self.Options.OutputCodecTypes
+
+	transopts := transcode.Options{}
+	transopts.FindAudioDecoderEncoder = func(codec av.AudioCodecData, i int) (ok bool, dec av.AudioDecoder, enc av.AudioEncoder, err error) {
+		if len(supports) == 0 {
+			return
+		}
+
+		support := false
+		for _, typ := range supports {
+			if typ == codec.Type() {
+				support = true
+			}
+		}
+
+		if support {
+			return
+		}
+		ok = true
+
+		var enctype av.CodecType
+		for _, typ := range supports {
+			if typ.IsAudio() {
+				if enc, _ = avutil.DefaultHandlers.NewAudioEncoder(typ); enc != nil {
+					enctype = typ
+					break
+				}
+			}
+		}
+		if enc == nil {
+			err = fmt.Errorf("avconv: convert %s->%s failed", codec.Type(), enctype)
+			return
+		}
+
+		// TODO: support per stream option
+		// enc.SetSampleRate ...
+
+		if dec, err = avutil.DefaultHandlers.NewAudioDecoder(codec); err != nil {
+			err = fmt.Errorf("avconv: decode %s failed", codec.Type())
+			return
+		}
+
+		return
+	}
+
+	self.transdemux = &transcode.Demuxer{
+		Options: transopts,
+		Demuxer: self.Demuxer,
+	}
+	if self.streams, err = self.transdemux.Streams(); err != nil {
+		return
+	}
+
+	return
+}
+
+func ConvertCmdline(args []string) (err error) {
+	output := ""
+	input := ""
+	flagi := false
+	flagv := false
+	flagt := false
+	flagre := false
+	duration := time.Duration(0)
+	options := Options{}
+
+	for _, arg := range args {
+		switch arg {
+		case "-i":
+			flagi = true
+
+		case "-v":
+			flagv = true
+
+		case "-t":
+			flagt = true
+
+		case "-re":
+			flagre = true
+
+		default:
+			switch {
+			case flagi:
+				flagi = false
+				input = arg
+
+			case flagt:
+				flagt = false
+				var f float64
+				fmt.Sscanf(arg, "%f", &f)
+				duration = time.Duration(f * float64(time.Second))
+
+			default:
+				output = arg
+			}
+		}
+	}
+
+	if input == "" {
+		err = fmt.Errorf("avconv: input file not specified")
+		return
+	}
+
+	if output == "" {
+		err = fmt.Errorf("avconv: output file not specified")
+		return
+	}
+
+	var demuxer av.DemuxCloser
+	var muxer av.MuxCloser
+
+	if demuxer, err = avutil.Open(input); err != nil {
+		return
+	}
+	defer demuxer.Close()
+
+	var handler avutil.RegisterHandler
+	if handler, muxer, err = avutil.DefaultHandlers.FindCreate(output); err != nil {
+		return
+	}
+	defer muxer.Close()
+
+	options.OutputCodecTypes = handler.CodecTypes
+
+	convdemux := &Demuxer{
+		Options: options,
+		Demuxer: demuxer,
+	}
+	defer convdemux.Close()
+
+	var streams []av.CodecData
+	if streams, err = demuxer.Streams(); err != nil {
+		return
+	}
+
+	var convstreams []av.CodecData
+	if convstreams, err = convdemux.Streams(); err != nil {
+		return
+	}
+
+	if flagv {
+		for _, stream := range streams {
+			fmt.Print(stream.Type(), " ")
+		}
+		fmt.Print("-> ")
+		for _, stream := range convstreams {
+			fmt.Print(stream.Type(), " ")
+		}
+		fmt.Println()
+	}
+
+	if err = muxer.WriteHeader(convstreams); err != nil {
+		return
+	}
+
+	filters := pktque.Filters{}
+	if flagre {
+		filters = append(filters, &pktque.Walltime{})
+	}
+	filterdemux := &pktque.FilterDemuxer{
+		Demuxer: convdemux,
+		Filter:  filters,
+	}
+
+	for {
+		var pkt av.Packet
+		if pkt, err = filterdemux.ReadPacket(); err != nil {
+			if err == io.EOF {
+				err = nil
+				break
+			}
+			return
+		}
+		if flagv {
+			fmt.Println(pkt.Idx, pkt.Time, len(pkt.Data), pkt.IsKeyFrame)
+		}
+		if duration != 0 && pkt.Time > duration {
+			break
+		}
+		if err = muxer.WritePacket(pkt); err != nil {
+			return
+		}
+	}
+
+	if err = muxer.WriteTrailer(); err != nil {
+		return
+	}
+
+	return
+}

av/avutil/avutil.go

@@ -0,0 +1,312 @@
+package avutil
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/deepch/vdk/av"
+)
+
+type HandlerDemuxer struct {
+	av.Demuxer
+	r io.ReadCloser
+}
+
+func (self *HandlerDemuxer) Close() error {
+	return self.r.Close()
+}
+
+type HandlerMuxer struct {
+	av.Muxer
+	w     io.WriteCloser
+	stage int
+}
+
+func (self *HandlerMuxer) WriteHeader(streams []av.CodecData) (err error) {
+	if self.stage == 0 {
+		if err = self.Muxer.WriteHeader(streams); err != nil {
+			return
+		}
+		self.stage++
+	}
+	return
+}
+
+func (self *HandlerMuxer) WriteTrailer() (err error) {
+	if self.stage == 1 {
+		self.stage++
+		if err = self.Muxer.WriteTrailer(); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (self *HandlerMuxer) Close() (err error) {
+	if err = self.WriteTrailer(); err != nil {
+		return
+	}
+	return self.w.Close()
+}
+
+type RegisterHandler struct {
+	Ext           string
+	ReaderDemuxer func(io.Reader) av.Demuxer
+	WriterMuxer   func(io.Writer) av.Muxer
+	UrlMuxer      func(string) (bool, av.MuxCloser, error)
+	UrlDemuxer    func(string) (bool, av.DemuxCloser, error)
+	UrlReader     func(string) (bool, io.ReadCloser, error)
+	Probe         func([]byte) bool
+	AudioEncoder  func(av.CodecType) (av.AudioEncoder, error)
+	AudioDecoder  func(av.AudioCodecData) (av.AudioDecoder, error)
+	ServerDemuxer func(string) (bool, av.DemuxCloser, error)
+	ServerMuxer   func(string) (bool, av.MuxCloser, error)
+	CodecTypes    []av.CodecType
+}
+
+type Handlers struct {
+	handlers []RegisterHandler
+}
+
+func (self *Handlers) Add(fn func(*RegisterHandler)) {
+	handler := &RegisterHandler{}
+	fn(handler)
+	self.handlers = append(self.handlers, *handler)
+}
+
+func (self *Handlers) openUrl(u *url.URL, uri string) (r io.ReadCloser, err error) {
+	if u != nil && u.Scheme != "" {
+		for _, handler := range self.handlers {
+			if handler.UrlReader != nil {
+				var ok bool
+				if ok, r, err = handler.UrlReader(uri); ok {
+					return
+				}
+			}
+		}
+		err = fmt.Errorf("avutil: openUrl %s failed", uri)
+	} else {
+		r, err = os.Open(uri)
+	}
+	return
+}
+
+func (self *Handlers) createUrl(u *url.URL, uri string) (w io.WriteCloser, err error) {
+	w, err = os.Create(uri)
+	return
+}
+
+func (self *Handlers) NewAudioEncoder(typ av.CodecType) (enc av.AudioEncoder, err error) {
+	for _, handler := range self.handlers {
+		if handler.AudioEncoder != nil {
+			if enc, _ = handler.AudioEncoder(typ); enc != nil {
+				return
+			}
+		}
+	}
+	err = fmt.Errorf("avutil: encoder", typ, "not found")
+	return
+}
+
+func (self *Handlers) NewAudioDecoder(codec av.AudioCodecData) (dec av.AudioDecoder, err error) {
+	for _, handler := range self.handlers {
+		if handler.AudioDecoder != nil {
+			if dec, _ = handler.AudioDecoder(codec); dec != nil {
+				return
+			}
+		}
+	}
+	err = fmt.Errorf("avutil: decoder", codec.Type(), "not found")
+	return
+}
+
+func (self *Handlers) Open(uri string) (demuxer av.DemuxCloser, err error) {
+	listen := false
+	if strings.HasPrefix(uri, "listen:") {
+		uri = uri[len("listen:"):]
+		listen = true
+	}
+
+	for _, handler := range self.handlers {
+		if listen {
+			if handler.ServerDemuxer != nil {
+				var ok bool
+				if ok, demuxer, err = handler.ServerDemuxer(uri); ok {
+					return
+				}
+			}
+		} else {
+			if handler.UrlDemuxer != nil {
+				var ok bool
+				if ok, demuxer, err = handler.UrlDemuxer(uri); ok {
+					return
+				}
+			}
+		}
+	}
+
+	var r io.ReadCloser
+	var ext string
+	var u *url.URL
+	if u, _ = url.Parse(uri); u != nil && u.Scheme != "" {
+		ext = path.Ext(u.Path)
+	} else {
+		ext = path.Ext(uri)
+	}
+
+	if ext != "" {
+		for _, handler := range self.handlers {
+			if handler.Ext == ext {
+				if handler.ReaderDemuxer != nil {
+					if r, err = self.openUrl(u, uri); err != nil {
+						return
+					}
+					demuxer = &HandlerDemuxer{
+						Demuxer: handler.ReaderDemuxer(r),
+						r:       r,
+					}
+					return
+				}
+			}
+		}
+	}
+
+	var probebuf [1024]byte
+	if r, err = self.openUrl(u, uri); err != nil {
+		return
+	}
+	if _, err = io.ReadFull(r, probebuf[:]); err != nil {
+		return
+	}
+
+	for _, handler := range self.handlers {
+		if handler.Probe != nil && handler.Probe(probebuf[:]) && handler.ReaderDemuxer != nil {
+			var _r io.Reader
+			if rs, ok := r.(io.ReadSeeker); ok {
+				if _, err = rs.Seek(0, 0); err != nil {
+					return
+				}
+				_r = rs
+			} else {
+				_r = io.MultiReader(bytes.NewReader(probebuf[:]), r)
+			}
+			demuxer = &HandlerDemuxer{
+				Demuxer: handler.ReaderDemuxer(_r),
+				r:       r,
+			}
+			return
+		}
+	}
+
+	r.Close()
+	err = fmt.Errorf("avutil: open %s failed", uri)
+	return
+}
+
+func (self *Handlers) Create(uri string) (muxer av.MuxCloser, err error) {
+	_, muxer, err = self.FindCreate(uri)
+	return
+}
+
+func (self *Handlers) FindCreate(uri string) (handler RegisterHandler, muxer av.MuxCloser, err error) {
+	listen := false
+	if strings.HasPrefix(uri, "listen:") {
+		uri = uri[len("listen:"):]
+		listen = true
+	}
+
+	for _, handler = range self.handlers {
+		if listen {
+			if handler.ServerMuxer != nil {
+				var ok bool
+				if ok, muxer, err = handler.ServerMuxer(uri); ok {
+					return
+				}
+			}
+		} else {
+			if handler.UrlMuxer != nil {
+				var ok bool
+				if ok, muxer, err = handler.UrlMuxer(uri); ok {
+					return
+				}
+			}
+		}
+	}
+
+	var ext string
+	var u *url.URL
+	if u, _ = url.Parse(uri); u != nil && u.Scheme != "" {
+		ext = path.Ext(u.Path)
+	} else {
+		ext = path.Ext(uri)
+	}
+
+	if ext != "" {
+		for _, handler = range self.handlers {
+			if handler.Ext == ext && handler.WriterMuxer != nil {
+				var w io.WriteCloser
+				if w, err = self.createUrl(u, uri); err != nil {
+					return
+				}
+				muxer = &HandlerMuxer{
+					Muxer: handler.WriterMuxer(w),
+					w:     w,
+				}
+				return
+			}
+		}
+	}
+
+	err = fmt.Errorf("avutil: create muxer %s failed", uri)
+	return
+}
+
+var DefaultHandlers = &Handlers{}
+
+func Open(url string) (demuxer av.DemuxCloser, err error) {
+	return DefaultHandlers.Open(url)
+}
+
+func Create(url string) (muxer av.MuxCloser, err error) {
+	return DefaultHandlers.Create(url)
+}
+
+func CopyPackets(dst av.PacketWriter, src av.PacketReader) (err error) {
+	for {
+		var pkt av.Packet
+		if pkt, err = src.ReadPacket(); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return
+		}
+		if err = dst.WritePacket(pkt); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func CopyFile(dst av.Muxer, src av.Demuxer) (err error) {
+	var streams []av.CodecData
+	if streams, err = src.Streams(); err != nil {
+		return
+	}
+	if err = dst.WriteHeader(streams); err != nil {
+		return
+	}
+	if err = CopyPackets(dst, src); err != nil {
+		if err != io.EOF {
+			return
+		}
+	}
+	if err = dst.WriteTrailer(); err != nil {
+		return
+	}
+	return
+}

av/pktque/buf.go

@@ -0,0 +1,73 @@
+package pktque
+
+import (
+	"github.com/deepch/vdk/av"
+)
+
+type Buf struct {
+	Head, Tail BufPos
+	pkts       []av.Packet
+	Size       int
+	Count      int
+}
+
+func NewBuf() *Buf {
+	return &Buf{
+		pkts: make([]av.Packet, 64),
+	}
+}
+
+func (self *Buf) Pop() av.Packet {
+	if self.Count == 0 {
+		panic("pktque.Buf: Pop() when count == 0")
+	}
+
+	i := int(self.Head) & (len(self.pkts) - 1)
+	pkt := self.pkts[i]
+	self.pkts[i] = av.Packet{}
+	self.Size -= len(pkt.Data)
+	self.Head++
+	self.Count--
+
+	return pkt
+}
+
+func (self *Buf) grow() {
+	newpkts := make([]av.Packet, len(self.pkts)*2)
+	for i := self.Head; i.LT(self.Tail); i++ {
+		newpkts[int(i)&(len(newpkts)-1)] = self.pkts[int(i)&(len(self.pkts)-1)]
+	}
+	self.pkts = newpkts
+}
+
+func (self *Buf) Push(pkt av.Packet) {
+	if self.Count == len(self.pkts) {
+		self.grow()
+	}
+	self.pkts[int(self.Tail)&(len(self.pkts)-1)] = pkt
+	self.Tail++
+	self.Count++
+	self.Size += len(pkt.Data)
+}
+
+func (self *Buf) Get(pos BufPos) av.Packet {
+	return self.pkts[int(pos)&(len(self.pkts)-1)]
+}
+
+func (self *Buf) IsValidPos(pos BufPos) bool {
+	return pos.GE(self.Head) && pos.LT(self.Tail)
+}
+
+type BufPos int
+
+func (self BufPos) LT(pos BufPos) bool {
+	return self-pos < 0
+}
+
+func (self BufPos) GE(pos BufPos) bool {
+	return self-pos >= 0
+}
+
+func (self BufPos) GT(pos BufPos) bool {
+	return self-pos > 0
+}

av/pktque/filters.go

@@ -0,0 +1,190 @@
+// Package pktque provides packet Filter interface and structures used by other components.
+package pktque
+
+import (
+	"time"
+
+	"github.com/deepch/vdk/av"
+)
+
+type Filter interface {
+	// Change packet time or drop packet
+	ModifyPacket(pkt *av.Packet, streams []av.CodecData, videoidx int, audioidx int) (drop bool, err error)
+}
+
+// Combine multiple Filters into one, ModifyPacket will be called in order.
+type Filters []Filter
+
+func (self Filters) ModifyPacket(pkt *av.Packet, streams []av.CodecData, videoidx int, audioidx int) (drop bool, err error) {
+	for _, filter := range self {
+		if drop, err = filter.ModifyPacket(pkt, streams, videoidx, audioidx); err != nil {
+			return
+		}
+		if drop {
+			return
+		}
+	}
+	return
+}
+
+// Wrap origin Demuxer and Filter into a new Demuxer, when read this Demuxer filters will be called.
+type FilterDemuxer struct {
+	av.Demuxer
+	Filter   Filter
+	streams  []av.CodecData
+	videoidx int
+	audioidx int
+}
+
+func (self FilterDemuxer) ReadPacket() (pkt av.Packet, err error) {
+	if self.streams == nil {
+		if self.streams, err = self.Demuxer.Streams(); err != nil {
+			return
+		}
+		for i, stream := range self.streams {
+			if stream.Type().IsVideo() {
+				self.videoidx = i
+			} else if stream.Type().IsAudio() {
+				self.audioidx = i
+			}
+		}
+	}
+
+	for {
+		if pkt, err = self.Demuxer.ReadPacket(); err != nil {
+			return
+		}
+		var drop bool
+		if drop, err = self.Filter.ModifyPacket(&pkt, self.streams, self.videoidx, self.audioidx); err != nil {
+			return
+		}
+		if !drop {
+			break
+		}
+	}
+
+	return
+}
+
+// Drop packets until first video key frame arrived.
+type WaitKeyFrame struct {
+	ok bool
+}
+
+func (self *WaitKeyFrame) ModifyPacket(pkt *av.Packet, streams []av.CodecData, videoidx int, audioidx int) (drop bool, err error) {
+	if !self.ok && pkt.Idx == int8(videoidx) && pkt.IsKeyFrame {
+		self.ok = true
+	}
+	drop = !self.ok
+	return
+}
+
+// Fix incorrect packet timestamps.
+type FixTime struct {
+	zerobase      time.Duration
+	incrbase      time.Duration
+	lasttime      time.Duration
+	StartFromZero bool // make timestamp start from zero
+	MakeIncrement bool // force timestamp increment
+}
+
+func (self *FixTime) ModifyPacket(pkt *av.Packet, streams []av.CodecData, videoidx int, audioidx int) (drop bool, err error) {
+	if self.StartFromZero {
+		if self.zerobase == 0 {
+			self.zerobase = pkt.Time
+		}
+		pkt.Time -= self.zerobase
+	}
+
+	if self.MakeIncrement {
+		pkt.Time -= self.incrbase
+		if self.lasttime == 0 {
+			self.lasttime = pkt.Time
+		}
+		if pkt.Time < self.lasttime || pkt.Time > self.lasttime+time.Millisecond*500 {
+			self.incrbase += pkt.Time - self.lasttime
+			pkt.Time = self.lasttime
+		}
+		self.lasttime = pkt.Time
+	}
+
+	return
+}
+
+// Drop incorrect packets to make A/V sync.
+type AVSync struct {
+	MaxTimeDiff time.Duration
+	time        []time.Duration
+}
+
+func (self *AVSync) ModifyPacket(pkt *av.Packet, streams []av.CodecData, videoidx int, audioidx int) (drop bool, err error) {
+	if self.time == nil {
+		self.time = make([]time.Duration, len(streams))
+		if self.MaxTimeDiff == 0 {
+			self.MaxTimeDiff = time.Millisecond * 500
+		}
+	}
+
+	start, end, correctable, correcttime := self.check(int(pkt.Idx))
+	if pkt.Time >= start && pkt.Time < end {
+		self.time[pkt.Idx] = pkt.Time
+	} else {
+		if correctable {
+			pkt.Time = correcttime
+			for i := range self.time {
+				self.time[i] = correcttime
+			}
+		} else {
+			drop = true
+		}
+	}
+	return
+}
+
+func (self *AVSync) check(i int) (start time.Duration, end time.Duration, correctable bool, correcttime time.Duration) {
+	minidx := -1
+	maxidx := -1
+	for j := range self.time {
+		if minidx == -1 || self.time[j] < self.time[minidx] {
+			minidx = j
+		}
+		if maxidx == -1 || self.time[j] > self.time[maxidx] {
+			maxidx = j
+		}
+	}
+	allthesame := self.time[minidx] == self.time[maxidx]
+
+	if i == maxidx {
+		if allthesame {
+			correctable = true
+		} else {
+			correctable = false
+		}
+	} else {
+		correctable = true
+	}
+
+	start = self.time[minidx]
+	end = start + self.MaxTimeDiff
+	correcttime = start + time.Millisecond*40
+	return
+}
+
+// Make packets reading speed as same as walltime, effect like ffmpeg -re option.
+type Walltime struct {
+	firsttime time.Time
+}
+
+func (self *Walltime) ModifyPacket(pkt *av.Packet, streams []av.CodecData, videoidx int, audioidx int) (drop bool, err error) {
+	if pkt.Idx == 0 {
+		if self.firsttime.IsZero() {
+			self.firsttime = time.Now()
+		}
+		pkttime := self.firsttime.Add(pkt.Time)
+		delta := pkttime.Sub(time.Now())
+		if delta > 0 {
+			time.Sleep(delta)
+		}
+	}
+	return
+}

av/pktque/timeline.go

@@ -0,0 +1,61 @@
+package pktque
+
+import (
+	"time"
+)
+
+/*
+pop                                   push
+
+     seg                 seg        seg
+  |--------|         |---------|   |---|
+     20ms                40ms       5ms
+----------------- time -------------------->
+headtm                               tailtm
+*/
+
+type tlSeg struct {
+	tm, dur time.Duration
+}
+
+type Timeline struct {
+	segs []tlSeg
+	headtm time.Duration
+}
+
+func (self *Timeline) Push(tm time.Duration, dur time.Duration) {
+	if len(self.segs) > 0 {
+		tail := self.segs[len(self.segs)-1]
+		diff := tm-(tail.tm+tail.dur)
+		if diff < 0 {
+			tm -= diff
+		}
+	}
+	self.segs = append(self.segs, tlSeg{tm, dur})
+}
+
+func (self *Timeline) Pop(dur time.Duration) (tm time.Duration) {
+	if len(self.segs) == 0 {
+		return self.headtm
+	}
+
+	tm = self.segs[0].tm
+	for dur > 0 && len(self.segs) > 0 {
+		seg := &self.segs[0]
+		sub := dur
+		if seg.dur < sub {
+			sub = seg.dur
+		}
+		seg.dur -= sub
+		dur -= sub
+		seg.tm += sub
+		self.headtm += sub
+		if seg.dur == 0 {
+			copy(self.segs[0:], self.segs[1:])
+			self.segs = self.segs[:len(self.segs)-1]
+		}
+	}
+
+	return
+}
+

av/pubsub/queue.go

@@ -0,0 +1,218 @@
+// Packege pubsub implements publisher-subscribers model used in multi-channel streaming.
+package pubsub
+
+import (
+	"io"
+	"sync"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/av/pktque"
+)
+
+//        time
+// ----------------->
+//
+// V-A-V-V-A-V-V-A-V-V
+// |                 |
+// 0        5        10
+// head             tail
+// oldest          latest
+//
+
+// One publisher and multiple subscribers thread-safe packet buffer queue.
+type Queue struct {
+	buf                      *pktque.Buf
+	head, tail               int
+	lock                     *sync.RWMutex
+	cond                     *sync.Cond
+	curgopcount, maxgopcount int
+	streams                  []av.CodecData
+	videoidx                 int
+	closed                   bool
+}
+
+func NewQueue() *Queue {
+	q := &Queue{}
+	q.buf = pktque.NewBuf()
+	q.maxgopcount = 2
+	q.lock = &sync.RWMutex{}
+	q.cond = sync.NewCond(q.lock.RLocker())
+	q.videoidx = -1
+	return q
+}
+
+func (self *Queue) SetMaxGopCount(n int) {
+	self.lock.Lock()
+	self.maxgopcount = n
+	self.lock.Unlock()
+	return
+}
+
+func (self *Queue) WriteHeader(streams []av.CodecData) error {
+	self.lock.Lock()
+
+	self.streams = streams
+	for i, stream := range streams {
+		if stream.Type().IsVideo() {
+			self.videoidx = i
+		}
+	}
+	self.cond.Broadcast()
+
+	self.lock.Unlock()
+
+	return nil
+}
+
+func (self *Queue) WriteTrailer() error {
+	return nil
+}
+
+// After Close() called, all QueueCursor's ReadPacket will return io.EOF.
+func (self *Queue) Close() (err error) {
+	self.lock.Lock()
+
+	self.closed = true
+	self.cond.Broadcast()
+
+	self.lock.Unlock()
+	return
+}
+
+// Put packet into buffer, old packets will be discared.
+func (self *Queue) WritePacket(pkt av.Packet) (err error) {
+	self.lock.Lock()
+
+	self.buf.Push(pkt)
+	if pkt.Idx == int8(self.videoidx) && pkt.IsKeyFrame {
+		self.curgopcount++
+	}
+
+	for self.curgopcount >= self.maxgopcount && self.buf.Count > 1 {
+		pkt := self.buf.Pop()
+		if pkt.Idx == int8(self.videoidx) && pkt.IsKeyFrame {
+			self.curgopcount--
+		}
+		if self.curgopcount < self.maxgopcount {
+			break
+		}
+	}
+	//println("shrink", self.curgopcount, self.maxgopcount, self.buf.Head, self.buf.Tail, "count", self.buf.Count, "size", self.buf.Size)
+
+	self.cond.Broadcast()
+
+	self.lock.Unlock()
+	return
+}
+
+type QueueCursor struct {
+	que    *Queue
+	pos    pktque.BufPos
+	gotpos bool
+	init   func(buf *pktque.Buf, videoidx int) pktque.BufPos
+}
+
+func (self *Queue) newCursor() *QueueCursor {
+	return &QueueCursor{
+		que: self,
+	}
+}
+
+// Create cursor position at latest packet.
+func (self *Queue) Latest() *QueueCursor {
+	cursor := self.newCursor()
+	cursor.init = func(buf *pktque.Buf, videoidx int) pktque.BufPos {
+		return buf.Tail
+	}
+	return cursor
+}
+
+// Create cursor position at oldest buffered packet.
+func (self *Queue) Oldest() *QueueCursor {
+	cursor := self.newCursor()
+	cursor.init = func(buf *pktque.Buf, videoidx int) pktque.BufPos {
+		return buf.Head
+	}
+	return cursor
+}
+
+// Create cursor position at specific time in buffered packets.
+func (self *Queue) DelayedTime(dur time.Duration) *QueueCursor {
+	cursor := self.newCursor()
+	cursor.init = func(buf *pktque.Buf, videoidx int) pktque.BufPos {
+		i := buf.Tail - 1
+		if buf.IsValidPos(i) {
+			end := buf.Get(i)
+			for buf.IsValidPos(i) {
+				if end.Time-buf.Get(i).Time > dur {
+					break
+				}
+				i--
+			}
+		}
+		return i
+	}
+	return cursor
+}
+
+// Create cursor position at specific delayed GOP count in buffered packets.
+func (self *Queue) DelayedGopCount(n int) *QueueCursor {
+	cursor := self.newCursor()
+	cursor.init = func(buf *pktque.Buf, videoidx int) pktque.BufPos {
+		i := buf.Tail - 1
+		if videoidx != -1 {
+			for gop := 0; buf.IsValidPos(i) && gop < n; i-- {
+				pkt := buf.Get(i)
+				if pkt.Idx == int8(self.videoidx) && pkt.IsKeyFrame {
+					gop++
+				}
+			}
+		}
+		return i
+	}
+	return cursor
+}
+
+func (self *QueueCursor) Streams() (streams []av.CodecData, err error) {
+	self.que.cond.L.Lock()
+	for self.que.streams == nil && !self.que.closed {
+		self.que.cond.Wait()
+	}
+	if self.que.streams != nil {
+		streams = self.que.streams
+	} else {
+		err = io.EOF
+	}
+	self.que.cond.L.Unlock()
+	return
+}
+
+// ReadPacket will not consume packets in Queue, it's just a cursor.
+func (self *QueueCursor) ReadPacket() (pkt av.Packet, err error) {
+	self.que.cond.L.Lock()
+	buf := self.que.buf
+	if !self.gotpos {
+		self.pos = self.init(buf, self.que.videoidx)
+		self.gotpos = true
+	}
+	for {
+		if self.pos.LT(buf.Head) {
+			self.pos = buf.Head
+		} else if self.pos.GT(buf.Tail) {
+			self.pos = buf.Tail
+		}
+		if buf.IsValidPos(self.pos) {
+			pkt = buf.Get(self.pos)
+			self.pos++
+			break
+		}
+		if self.que.closed {
+			err = io.EOF
+			break
+		}
+		self.que.cond.Wait()
+	}
+	self.que.cond.L.Unlock()
+	return
+}

av/transcode/transcode.go

@@ -0,0 +1,247 @@
+// Package transcoder implements Transcoder based on Muxer/Demuxer and AudioEncoder/AudioDecoder interface.
+package transcode
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/av/pktque"
+)
+
+var Debug bool
+
+type tStream struct {
+	codec              av.CodecData
+	timeline           *pktque.Timeline
+	aencodec, adecodec av.AudioCodecData
+	aenc               av.AudioEncoder
+	adec               av.AudioDecoder
+}
+
+type Options struct {
+	// check if transcode is needed, and create the AudioDecoder and AudioEncoder.
+	FindAudioDecoderEncoder func(codec av.AudioCodecData, i int) (
+		need bool, dec av.AudioDecoder, enc av.AudioEncoder, err error,
+	)
+}
+
+type Transcoder struct {
+	streams []*tStream
+}
+
+func NewTranscoder(streams []av.CodecData, options Options) (_self *Transcoder, err error) {
+	self := &Transcoder{}
+	self.streams = []*tStream{}
+
+	for i, stream := range streams {
+		ts := &tStream{codec: stream}
+		if stream.Type().IsAudio() {
+			if options.FindAudioDecoderEncoder != nil {
+				var ok bool
+				var enc av.AudioEncoder
+				var dec av.AudioDecoder
+				ok, dec, enc, err = options.FindAudioDecoderEncoder(stream.(av.AudioCodecData), i)
+				if ok {
+					if err != nil {
+						return
+					}
+					ts.timeline = &pktque.Timeline{}
+					if ts.codec, err = enc.CodecData(); err != nil {
+						return
+					}
+					ts.aencodec = ts.codec.(av.AudioCodecData)
+					ts.adecodec = stream.(av.AudioCodecData)
+					ts.aenc = enc
+					ts.adec = dec
+				}
+			}
+		}
+		self.streams = append(self.streams, ts)
+	}
+
+	_self = self
+	return
+}
+
+func (self *tStream) audioDecodeAndEncode(inpkt av.Packet) (outpkts []av.Packet, err error) {
+	var dur time.Duration
+	var frame av.AudioFrame
+	var ok bool
+	if ok, frame, err = self.adec.Decode(inpkt.Data); err != nil {
+		return
+	}
+	if !ok {
+		return
+	}
+
+	if dur, err = self.adecodec.PacketDuration(inpkt.Data); err != nil {
+		err = fmt.Errorf("transcode: PacketDuration() failed for input stream #%d", inpkt.Idx)
+		return
+	}
+
+	if Debug {
+		fmt.Println("transcode: push", inpkt.Time, dur)
+	}
+	self.timeline.Push(inpkt.Time, dur)
+
+	var _outpkts [][]byte
+	if _outpkts, err = self.aenc.Encode(frame); err != nil {
+		return
+	}
+	for _, _outpkt := range _outpkts {
+		if dur, err = self.aencodec.PacketDuration(_outpkt); err != nil {
+			err = fmt.Errorf("transcode: PacketDuration() failed for output stream #%d", inpkt.Idx)
+			return
+		}
+		outpkt := av.Packet{Idx: inpkt.Idx, Data: _outpkt}
+		outpkt.Time = self.timeline.Pop(dur)
+
+		if Debug {
+			fmt.Println("transcode: pop", outpkt.Time, dur)
+		}
+
+		outpkts = append(outpkts, outpkt)
+	}
+
+	return
+}
+
+// Do the transcode.
+//
+// In audio transcoding one Packet may transcode into many Packets
+// packet time will be adjusted automatically.
+func (self *Transcoder) Do(pkt av.Packet) (out []av.Packet, err error) {
+	stream := self.streams[pkt.Idx]
+	if stream.aenc != nil && stream.adec != nil {
+		if out, err = stream.audioDecodeAndEncode(pkt); err != nil {
+			return
+		}
+	} else {
+		out = append(out, pkt)
+	}
+	return
+}
+
+// Get CodecDatas after transcoding.
+func (self *Transcoder) Streams() (streams []av.CodecData, err error) {
+	for _, stream := range self.streams {
+		streams = append(streams, stream.codec)
+	}
+	return
+}
+
+// Close transcoder, close related encoder and decoders.
+func (self *Transcoder) Close() (err error) {
+	for _, stream := range self.streams {
+		if stream.aenc != nil {
+			stream.aenc.Close()
+			stream.aenc = nil
+		}
+		if stream.adec != nil {
+			stream.adec.Close()
+			stream.adec = nil
+		}
+	}
+	self.streams = nil
+	return
+}
+
+// Wrap transcoder and origin Muxer into new Muxer.
+// Write to new Muxer will do transcoding automatically.
+type Muxer struct {
+	av.Muxer   // origin Muxer
+	Options    // transcode options
+	transcoder *Transcoder
+}
+
+func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) {
+	if self.transcoder, err = NewTranscoder(streams, self.Options); err != nil {
+		return
+	}
+	var newstreams []av.CodecData
+	if newstreams, err = self.transcoder.Streams(); err != nil {
+		return
+	}
+	if err = self.Muxer.WriteHeader(newstreams); err != nil {
+		return
+	}
+	return
+}
+
+func (self *Muxer) WritePacket(pkt av.Packet) (err error) {
+	var outpkts []av.Packet
+	if outpkts, err = self.transcoder.Do(pkt); err != nil {
+		return
+	}
+	for _, pkt := range outpkts {
+		if err = self.Muxer.WritePacket(pkt); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (self *Muxer) Close() (err error) {
+	if self.transcoder != nil {
+		return self.transcoder.Close()
+	}
+	return
+}
+
+// Wrap transcoder and origin Demuxer into new Demuxer.
+// Read this Demuxer will do transcoding automatically.
+type Demuxer struct {
+	av.Demuxer
+	Options
+	transcoder *Transcoder
+	outpkts    []av.Packet
+}
+
+func (self *Demuxer) prepare() (err error) {
+	if self.transcoder == nil {
+		var streams []av.CodecData
+		if streams, err = self.Demuxer.Streams(); err != nil {
+			return
+		}
+		if self.transcoder, err = NewTranscoder(streams, self.Options); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (self *Demuxer) ReadPacket() (pkt av.Packet, err error) {
+	if err = self.prepare(); err != nil {
+		return
+	}
+	for {
+		if len(self.outpkts) > 0 {
+			pkt = self.outpkts[0]
+			self.outpkts = self.outpkts[1:]
+			return
+		}
+		var rpkt av.Packet
+		if rpkt, err = self.Demuxer.ReadPacket(); err != nil {
+			return
+		}
+		if self.outpkts, err = self.transcoder.Do(rpkt); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (self *Demuxer) Streams() (streams []av.CodecData, err error) {
+	if err = self.prepare(); err != nil {
+		return
+	}
+	return self.transcoder.Streams()
+}
+
+func (self *Demuxer) Close() (err error) {
+	if self.transcoder != nil {
+		return self.transcoder.Close()
+	}
+	return
+}

cgo/ffmpeg/audio.go

@@ -0,0 +1,758 @@
+package ffmpeg
+
+/*
+#include "ffmpeg.h"
+int wrap_avcodec_decode_audio4(AVCodecContext *ctx, AVFrame *frame, void *data, int size, int *got) {
+	struct AVPacket pkt = {.data = data, .size = size};
+	return avcodec_decode_audio4(ctx, frame, got, &pkt);
+}
+int wrap_avresample_convert(AVAudioResampleContext *avr, int *out, int outsize, int outcount, int *in, int insize, int incount) {
+	return avresample_convert(avr, (void *)out, outsize, outcount, (void *)in, insize, incount);
+}
+*/
+import "C"
+import (
+	"fmt"
+	"runtime"
+	"time"
+	"unsafe"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/av/avutil"
+	"github.com/deepch/vdk/codec/aacparser"
+)
+
+const debug = false
+
+type Resampler struct {
+	inSampleFormat, OutSampleFormat   av.SampleFormat
+	inChannelLayout, OutChannelLayout av.ChannelLayout
+	inSampleRate, OutSampleRate       int
+	avr                               *C.AVAudioResampleContext
+}
+
+func (self *Resampler) Resample(in av.AudioFrame) (out av.AudioFrame, err error) {
+	formatChange := in.SampleRate != self.inSampleRate || in.SampleFormat != self.inSampleFormat || in.ChannelLayout != self.inChannelLayout
+
+	var flush av.AudioFrame
+
+	if formatChange {
+		if self.avr != nil {
+			outChannels := self.OutChannelLayout.Count()
+			if !self.OutSampleFormat.IsPlanar() {
+				outChannels = 1
+			}
+			outData := make([]*C.uint8_t, outChannels)
+			outSampleCount := int(C.avresample_get_out_samples(self.avr, C.int(in.SampleCount)))
+			outLinesize := outSampleCount * self.OutSampleFormat.BytesPerSample()
+			flush.Data = make([][]byte, outChannels)
+			for i := 0; i < outChannels; i++ {
+				flush.Data[i] = make([]byte, outLinesize)
+				outData[i] = (*C.uint8_t)(unsafe.Pointer(&flush.Data[i][0]))
+			}
+			flush.ChannelLayout = self.OutChannelLayout
+			flush.SampleFormat = self.OutSampleFormat
+			flush.SampleRate = self.OutSampleRate
+
+			convertSamples := int(C.wrap_avresample_convert(
+				self.avr,
+				(*C.int)(unsafe.Pointer(&outData[0])), C.int(outLinesize), C.int(outSampleCount),
+				nil, C.int(0), C.int(0),
+			))
+			if convertSamples < 0 {
+				err = fmt.Errorf("ffmpeg: avresample_convert_frame failed")
+				return
+			}
+			flush.SampleCount = convertSamples
+			if convertSamples < outSampleCount {
+				for i := 0; i < outChannels; i++ {
+					flush.Data[i] = flush.Data[i][:convertSamples*self.OutSampleFormat.BytesPerSample()]
+				}
+			}
+
+			//fmt.Println("flush:", "outSampleCount", outSampleCount, "convertSamples", convertSamples, "datasize", len(flush.Data[0]))
+		} else {
+			runtime.SetFinalizer(self, func(self *Resampler) {
+				self.Close()
+			})
+		}
+
+		C.avresample_free(&self.avr)
+		self.inSampleFormat = in.SampleFormat
+		self.inSampleRate = in.SampleRate
+		self.inChannelLayout = in.ChannelLayout
+		avr := C.avresample_alloc_context()
+		C.av_opt_set_int(unsafe.Pointer(avr), C.CString("in_channel_layout"), C.int64_t(channelLayoutAV2FF(self.inChannelLayout)), 0)
+		C.av_opt_set_int(unsafe.Pointer(avr), C.CString("out_channel_layout"), C.int64_t(channelLayoutAV2FF(self.OutChannelLayout)), 0)
+		C.av_opt_set_int(unsafe.Pointer(avr), C.CString("in_sample_rate"), C.int64_t(self.inSampleRate), 0)
+		C.av_opt_set_int(unsafe.Pointer(avr), C.CString("out_sample_rate"), C.int64_t(self.OutSampleRate), 0)
+		C.av_opt_set_int(unsafe.Pointer(avr), C.CString("in_sample_fmt"), C.int64_t(sampleFormatAV2FF(self.inSampleFormat)), 0)
+		C.av_opt_set_int(unsafe.Pointer(avr), C.CString("out_sample_fmt"), C.int64_t(sampleFormatAV2FF(self.OutSampleFormat)), 0)
+		C.avresample_open(avr)
+		self.avr = avr
+	}
+
+	var inChannels, inLinesize int
+	inSampleCount := in.SampleCount
+	if !self.inSampleFormat.IsPlanar() {
+		inChannels = 1
+		inLinesize = inSampleCount * in.SampleFormat.BytesPerSample() * self.inChannelLayout.Count()
+	} else {
+		inChannels = self.inChannelLayout.Count()
+		inLinesize = inSampleCount * in.SampleFormat.BytesPerSample()
+	}
+	inData := make([]*C.uint8_t, inChannels)
+	for i := 0; i < inChannels; i++ {
+		inData[i] = (*C.uint8_t)(unsafe.Pointer(&in.Data[i][0]))
+	}
+
+	var outChannels, outLinesize, outBytesPerSample int
+	outSampleCount := int(C.avresample_get_out_samples(self.avr, C.int(in.SampleCount)))
+	if !self.OutSampleFormat.IsPlanar() {
+		outChannels = 1
+		outBytesPerSample = self.OutSampleFormat.BytesPerSample() * self.OutChannelLayout.Count()
+		outLinesize = outSampleCount * outBytesPerSample
+	} else {
+		outChannels = self.OutChannelLayout.Count()
+		outBytesPerSample = self.OutSampleFormat.BytesPerSample()
+		outLinesize = outSampleCount * outBytesPerSample
+	}
+	outData := make([]*C.uint8_t, outChannels)
+	out.Data = make([][]byte, outChannels)
+	for i := 0; i < outChannels; i++ {
+		out.Data[i] = make([]byte, outLinesize)
+		outData[i] = (*C.uint8_t)(unsafe.Pointer(&out.Data[i][0]))
+	}
+	out.ChannelLayout = self.OutChannelLayout
+	out.SampleFormat = self.OutSampleFormat
+	out.SampleRate = self.OutSampleRate
+
+	convertSamples := int(C.wrap_avresample_convert(
+		self.avr,
+		(*C.int)(unsafe.Pointer(&outData[0])), C.int(outLinesize), C.int(outSampleCount),
+		(*C.int)(unsafe.Pointer(&inData[0])), C.int(inLinesize), C.int(inSampleCount),
+	))
+	if convertSamples < 0 {
+		err = fmt.Errorf("ffmpeg: avresample_convert_frame failed")
+		return
+	}
+
+	out.SampleCount = convertSamples
+	if convertSamples < outSampleCount {
+		for i := 0; i < outChannels; i++ {
+			out.Data[i] = out.Data[i][:convertSamples*outBytesPerSample]
+		}
+	}
+
+	if flush.SampleCount > 0 {
+		out = flush.Concat(out)
+	}
+
+	return
+}
+
+func (self *Resampler) Close() {
+	C.avresample_free(&self.avr)
+}
+
+type AudioEncoder struct {
+	ff               *ffctx
+	SampleRate       int
+	Bitrate          int
+	ChannelLayout    av.ChannelLayout
+	SampleFormat     av.SampleFormat
+	FrameSampleCount int
+	framebuf         av.AudioFrame
+	codecData        av.AudioCodecData
+	resampler        *Resampler
+}
+
+func sampleFormatAV2FF(sampleFormat av.SampleFormat) (ffsamplefmt int32) {
+	switch sampleFormat {
+	case av.U8:
+		ffsamplefmt = C.AV_SAMPLE_FMT_U8
+	case av.S16:
+		ffsamplefmt = C.AV_SAMPLE_FMT_S16
+	case av.S32:
+		ffsamplefmt = C.AV_SAMPLE_FMT_S32
+	case av.FLT:
+		ffsamplefmt = C.AV_SAMPLE_FMT_FLT
+	case av.DBL:
+		ffsamplefmt = C.AV_SAMPLE_FMT_DBL
+	case av.U8P:
+		ffsamplefmt = C.AV_SAMPLE_FMT_U8P
+	case av.S16P:
+		ffsamplefmt = C.AV_SAMPLE_FMT_S16P
+	case av.S32P:
+		ffsamplefmt = C.AV_SAMPLE_FMT_S32P
+	case av.FLTP:
+		ffsamplefmt = C.AV_SAMPLE_FMT_FLTP
+	case av.DBLP:
+		ffsamplefmt = C.AV_SAMPLE_FMT_DBLP
+	}
+	return
+}
+
+func sampleFormatFF2AV(ffsamplefmt int32) (sampleFormat av.SampleFormat) {
+	switch ffsamplefmt {
+	case C.AV_SAMPLE_FMT_U8: ///< unsigned 8 bits
+		sampleFormat = av.U8
+	case C.AV_SAMPLE_FMT_S16: ///< signed 16 bits
+		sampleFormat = av.S16
+	case C.AV_SAMPLE_FMT_S32: ///< signed 32 bits
+		sampleFormat = av.S32
+	case C.AV_SAMPLE_FMT_FLT: ///< float
+		sampleFormat = av.FLT
+	case C.AV_SAMPLE_FMT_DBL: ///< double
+		sampleFormat = av.DBL
+	case C.AV_SAMPLE_FMT_U8P: ///< unsigned 8 bits, planar
+		sampleFormat = av.U8P
+	case C.AV_SAMPLE_FMT_S16P: ///< signed 16 bits, planar
+		sampleFormat = av.S16P
+	case C.AV_SAMPLE_FMT_S32P: ///< signed 32 bits, planar
+		sampleFormat = av.S32P
+	case C.AV_SAMPLE_FMT_FLTP: ///< float, planar
+		sampleFormat = av.FLTP
+	case C.AV_SAMPLE_FMT_DBLP: ///< double, planar
+		sampleFormat = av.DBLP
+	}
+	return
+}
+
+func (self *AudioEncoder) SetSampleFormat(fmt av.SampleFormat) (err error) {
+	self.SampleFormat = fmt
+	return
+}
+
+func (self *AudioEncoder) SetSampleRate(rate int) (err error) {
+	self.SampleRate = rate
+	return
+}
+
+func (self *AudioEncoder) SetChannelLayout(ch av.ChannelLayout) (err error) {
+	self.ChannelLayout = ch
+	return
+}
+
+func (self *AudioEncoder) SetBitrate(bitrate int) (err error) {
+	self.Bitrate = bitrate
+	return
+}
+
+func (self *AudioEncoder) SetOption(key string, val interface{}) (err error) {
+	ff := &self.ff.ff
+
+	sval := fmt.Sprint(val)
+	if key == "profile" {
+		ff.profile = C.avcodec_profile_name_to_int(ff.codec, C.CString(sval))
+		if ff.profile == C.FF_PROFILE_UNKNOWN {
+			err = fmt.Errorf("ffmpeg: profile `%s` invalid", sval)
+			return
+		}
+		return
+	}
+
+	C.av_dict_set(&ff.options, C.CString(key), C.CString(sval), 0)
+	return
+}
+
+func (self *AudioEncoder) GetOption(key string, val interface{}) (err error) {
+	ff := &self.ff.ff
+	entry := C.av_dict_get(ff.options, C.CString(key), nil, 0)
+	if entry == nil {
+		err = fmt.Errorf("ffmpeg: GetOption failed: `%s` not exists", key)
+		return
+	}
+	switch p := val.(type) {
+	case *string:
+		*p = C.GoString(entry.value)
+	case *int:
+		fmt.Sscanf(C.GoString(entry.value), "%d", p)
+	default:
+		err = fmt.Errorf("ffmpeg: GetOption failed: val must be *string or *int receiver")
+		return
+	}
+	return
+}
+
+func (self *AudioEncoder) Setup() (err error) {
+	ff := &self.ff.ff
+
+	ff.frame = C.av_frame_alloc()
+
+	if self.SampleFormat == av.SampleFormat(0) {
+		self.SampleFormat = sampleFormatFF2AV(*ff.codec.sample_fmts)
+	}
+
+	//if self.Bitrate == 0 {
+	//	self.Bitrate = 80000
+	//}
+	if self.SampleRate == 0 {
+		self.SampleRate = 44100
+	}
+	if self.ChannelLayout == av.ChannelLayout(0) {
+		self.ChannelLayout = av.CH_STEREO
+	}
+
+	ff.codecCtx.sample_fmt = sampleFormatAV2FF(self.SampleFormat)
+	ff.codecCtx.sample_rate = C.int(self.SampleRate)
+	ff.codecCtx.bit_rate = C.int64_t(self.Bitrate)
+	ff.codecCtx.channel_layout = channelLayoutAV2FF(self.ChannelLayout)
+	ff.codecCtx.strict_std_compliance = C.FF_COMPLIANCE_EXPERIMENTAL
+	ff.codecCtx.flags = C.AV_CODEC_FLAG_GLOBAL_HEADER
+	ff.codecCtx.profile = ff.profile
+
+	if C.avcodec_open2(ff.codecCtx, ff.codec, nil) != 0 {
+		err = fmt.Errorf("ffmpeg: encoder: avcodec_open2 failed")
+		return
+	}
+	self.SampleFormat = sampleFormatFF2AV(ff.codecCtx.sample_fmt)
+	self.FrameSampleCount = int(ff.codecCtx.frame_size)
+
+	extradata := C.GoBytes(unsafe.Pointer(ff.codecCtx.extradata), ff.codecCtx.extradata_size)
+
+	switch ff.codecCtx.codec_id {
+	case C.AV_CODEC_ID_AAC:
+		if self.codecData, err = aacparser.NewCodecDataFromMPEG4AudioConfigBytes(extradata); err != nil {
+			return
+		}
+
+	default:
+		self.codecData = audioCodecData{
+			channelLayout: self.ChannelLayout,
+			sampleFormat:  self.SampleFormat,
+			sampleRate:    self.SampleRate,
+			codecId:       ff.codecCtx.codec_id,
+			extradata:     extradata,
+		}
+	}
+
+	return
+}
+
+func (self *AudioEncoder) prepare() (err error) {
+	ff := &self.ff.ff
+
+	if ff.frame == nil {
+		if err = self.Setup(); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (self *AudioEncoder) CodecData() (codec av.AudioCodecData, err error) {
+	if err = self.prepare(); err != nil {
+		return
+	}
+	codec = self.codecData
+	return
+}
+
+func (self *AudioEncoder) encodeOne(frame av.AudioFrame) (gotpkt bool, pkt []byte, err error) {
+	if err = self.prepare(); err != nil {
+		return
+	}
+
+	ff := &self.ff.ff
+
+	cpkt := C.AVPacket{}
+	cgotpkt := C.int(0)
+	audioFrameAssignToFF(frame, ff.frame)
+
+	if false {
+		farr := []string{}
+		for i := 0; i < len(frame.Data[0])/4; i++ {
+			var f *float64 = (*float64)(unsafe.Pointer(&frame.Data[0][i*4]))
+			farr = append(farr, fmt.Sprintf("%.8f", *f))
+		}
+		fmt.Println(farr)
+	}
+	cerr := C.avcodec_encode_audio2(ff.codecCtx, &cpkt, ff.frame, &cgotpkt)
+	if cerr < C.int(0) {
+		err = fmt.Errorf("ffmpeg: avcodec_encode_audio2 failed: %d", cerr)
+		return
+	}
+
+	if cgotpkt != 0 {
+		gotpkt = true
+		pkt = C.GoBytes(unsafe.Pointer(cpkt.data), cpkt.size)
+		C.av_packet_unref(&cpkt)
+
+		if debug {
+			fmt.Println("ffmpeg: Encode", frame.SampleCount, frame.SampleRate, frame.ChannelLayout, frame.SampleFormat, "len", len(pkt))
+		}
+	}
+
+	return
+}
+
+func (self *AudioEncoder) resample(in av.AudioFrame) (out av.AudioFrame, err error) {
+	if self.resampler == nil {
+		self.resampler = &Resampler{
+			OutSampleFormat:  self.SampleFormat,
+			OutSampleRate:    self.SampleRate,
+			OutChannelLayout: self.ChannelLayout,
+		}
+	}
+	if out, err = self.resampler.Resample(in); err != nil {
+		return
+	}
+	return
+}
+
+func (self *AudioEncoder) Encode(frame av.AudioFrame) (pkts [][]byte, err error) {
+	var gotpkt bool
+	var pkt []byte
+
+	if frame.SampleFormat != self.SampleFormat || frame.ChannelLayout != self.ChannelLayout || frame.SampleRate != self.SampleRate {
+		if frame, err = self.resample(frame); err != nil {
+			return
+		}
+	}
+
+	if self.FrameSampleCount != 0 {
+		if self.framebuf.SampleCount == 0 {
+			self.framebuf = frame
+		} else {
+			self.framebuf = self.framebuf.Concat(frame)
+		}
+		for self.framebuf.SampleCount >= self.FrameSampleCount {
+			frame := self.framebuf.Slice(0, self.FrameSampleCount)
+			if gotpkt, pkt, err = self.encodeOne(frame); err != nil {
+				return
+			}
+			if gotpkt {
+				pkts = append(pkts, pkt)
+			}
+			self.framebuf = self.framebuf.Slice(self.FrameSampleCount, self.framebuf.SampleCount)
+		}
+	} else {
+		if gotpkt, pkt, err = self.encodeOne(frame); err != nil {
+			return
+		}
+		if gotpkt {
+			pkts = append(pkts, pkt)
+		}
+	}
+
+	return
+}
+
+func (self *AudioEncoder) Close() {
+	freeFFCtx(self.ff)
+	if self.resampler != nil {
+		self.resampler.Close()
+		self.resampler = nil
+	}
+}
+
+func audioFrameAssignToAVParams(f *C.AVFrame, frame *av.AudioFrame) {
+	frame.SampleFormat = sampleFormatFF2AV(int32(f.format))
+	frame.ChannelLayout = channelLayoutFF2AV(f.channel_layout)
+	frame.SampleRate = int(f.sample_rate)
+}
+
+func audioFrameAssignToAVData(f *C.AVFrame, frame *av.AudioFrame) {
+	frame.SampleCount = int(f.nb_samples)
+	frame.Data = make([][]byte, int(f.channels))
+	for i := 0; i < int(f.channels); i++ {
+		frame.Data[i] = C.GoBytes(unsafe.Pointer(f.data[i]), f.linesize[0])
+	}
+}
+
+func audioFrameAssignToAV(f *C.AVFrame, frame *av.AudioFrame) {
+	audioFrameAssignToAVParams(f, frame)
+	audioFrameAssignToAVData(f, frame)
+}
+
+func audioFrameAssignToFFParams(frame av.AudioFrame, f *C.AVFrame) {
+	f.format = C.int(sampleFormatAV2FF(frame.SampleFormat))
+	f.channel_layout = channelLayoutAV2FF(frame.ChannelLayout)
+	f.sample_rate = C.int(frame.SampleRate)
+	f.channels = C.int(frame.ChannelLayout.Count())
+}
+
+func audioFrameAssignToFFData(frame av.AudioFrame, f *C.AVFrame) {
+	f.nb_samples = C.int(frame.SampleCount)
+	for i := range frame.Data {
+		f.data[i] = (*C.uint8_t)(unsafe.Pointer(&frame.Data[i][0]))
+		f.linesize[i] = C.int(len(frame.Data[i]))
+	}
+}
+
+func audioFrameAssignToFF(frame av.AudioFrame, f *C.AVFrame) {
+	audioFrameAssignToFFParams(frame, f)
+	audioFrameAssignToFFData(frame, f)
+}
+
+func channelLayoutFF2AV(layout C.uint64_t) (channelLayout av.ChannelLayout) {
+	if layout&C.AV_CH_FRONT_CENTER != 0 {
+		channelLayout |= av.CH_FRONT_CENTER
+	}
+	if layout&C.AV_CH_FRONT_LEFT != 0 {
+		channelLayout |= av.CH_FRONT_LEFT
+	}
+	if layout&C.AV_CH_FRONT_RIGHT != 0 {
+		channelLayout |= av.CH_FRONT_RIGHT
+	}
+	if layout&C.AV_CH_BACK_CENTER != 0 {
+		channelLayout |= av.CH_BACK_CENTER
+	}
+	if layout&C.AV_CH_BACK_LEFT != 0 {
+		channelLayout |= av.CH_BACK_LEFT
+	}
+	if layout&C.AV_CH_BACK_RIGHT != 0 {
+		channelLayout |= av.CH_BACK_RIGHT
+	}
+	if layout&C.AV_CH_SIDE_LEFT != 0 {
+		channelLayout |= av.CH_SIDE_LEFT
+	}
+	if layout&C.AV_CH_SIDE_RIGHT != 0 {
+		channelLayout |= av.CH_SIDE_RIGHT
+	}
+	if layout&C.AV_CH_LOW_FREQUENCY != 0 {
+		channelLayout |= av.CH_LOW_FREQ
+	}
+	return
+}
+
+func channelLayoutAV2FF(channelLayout av.ChannelLayout) (layout C.uint64_t) {
+	if channelLayout&av.CH_FRONT_CENTER != 0 {
+		layout |= C.AV_CH_FRONT_CENTER
+	}
+	if channelLayout&av.CH_FRONT_LEFT != 0 {
+		layout |= C.AV_CH_FRONT_LEFT
+	}
+	if channelLayout&av.CH_FRONT_RIGHT != 0 {
+		layout |= C.AV_CH_FRONT_RIGHT
+	}
+	if channelLayout&av.CH_BACK_CENTER != 0 {
+		layout |= C.AV_CH_BACK_CENTER
+	}
+	if channelLayout&av.CH_BACK_LEFT != 0 {
+		layout |= C.AV_CH_BACK_LEFT
+	}
+	if channelLayout&av.CH_BACK_RIGHT != 0 {
+		layout |= C.AV_CH_BACK_RIGHT
+	}
+	if channelLayout&av.CH_SIDE_LEFT != 0 {
+		layout |= C.AV_CH_SIDE_LEFT
+	}
+	if channelLayout&av.CH_SIDE_RIGHT != 0 {
+		layout |= C.AV_CH_SIDE_RIGHT
+	}
+	if channelLayout&av.CH_LOW_FREQ != 0 {
+		layout |= C.AV_CH_LOW_FREQUENCY
+	}
+	return
+}
+
+type AudioDecoder struct {
+	ff            *ffctx
+	ChannelLayout av.ChannelLayout
+	SampleFormat  av.SampleFormat
+	SampleRate    int
+	Extradata     []byte
+}
+
+func (self *AudioDecoder) Setup() (err error) {
+	ff := &self.ff.ff
+
+	ff.frame = C.av_frame_alloc()
+
+	if len(self.Extradata) > 0 {
+		ff.codecCtx.extradata = (*C.uint8_t)(unsafe.Pointer(&self.Extradata[0]))
+		ff.codecCtx.extradata_size = C.int(len(self.Extradata))
+	}
+	if debug {
+		fmt.Println("ffmpeg: Decoder.Setup Extradata.len", len(self.Extradata))
+	}
+
+	ff.codecCtx.sample_rate = C.int(self.SampleRate)
+	ff.codecCtx.channel_layout = channelLayoutAV2FF(self.ChannelLayout)
+	ff.codecCtx.channels = C.int(self.ChannelLayout.Count())
+	if C.avcodec_open2(ff.codecCtx, ff.codec, nil) != 0 {
+		err = fmt.Errorf("ffmpeg: decoder: avcodec_open2 failed")
+		return
+	}
+	self.SampleFormat = sampleFormatFF2AV(ff.codecCtx.sample_fmt)
+	self.ChannelLayout = channelLayoutFF2AV(ff.codecCtx.channel_layout)
+	if self.SampleRate == 0 {
+		self.SampleRate = int(ff.codecCtx.sample_rate)
+	}
+
+	return
+}
+
+func (self *AudioDecoder) Decode(pkt []byte) (gotframe bool, frame av.AudioFrame, err error) {
+	ff := &self.ff.ff
+
+	cgotframe := C.int(0)
+	cerr := C.wrap_avcodec_decode_audio4(ff.codecCtx, ff.frame, unsafe.Pointer(&pkt[0]), C.int(len(pkt)), &cgotframe)
+	if cerr < C.int(0) {
+		err = fmt.Errorf("ffmpeg: avcodec_decode_audio4 failed: %d", cerr)
+		return
+	}
+
+	if cgotframe != C.int(0) {
+		gotframe = true
+		audioFrameAssignToAV(ff.frame, &frame)
+		frame.SampleRate = self.SampleRate
+
+		if debug {
+			fmt.Println("ffmpeg: Decode", frame.SampleCount, frame.SampleRate, frame.ChannelLayout, frame.SampleFormat)
+		}
+	}
+
+	return
+}
+
+func (self *AudioDecoder) Close() {
+	freeFFCtx(self.ff)
+}
+
+func NewAudioEncoderByCodecType(typ av.CodecType) (enc *AudioEncoder, err error) {
+	var id uint32
+
+	switch typ {
+	case av.AAC:
+		id = C.AV_CODEC_ID_AAC
+
+	default:
+		err = fmt.Errorf("ffmpeg: cannot find encoder codecType=%d", typ)
+		return
+	}
+
+	codec := C.avcodec_find_encoder(id)
+	if codec == nil || C.avcodec_get_type(id) != C.AVMEDIA_TYPE_AUDIO {
+		err = fmt.Errorf("ffmpeg: cannot find audio encoder codecId=%d", id)
+		return
+	}
+
+	_enc := &AudioEncoder{}
+	if _enc.ff, err = newFFCtxByCodec(codec); err != nil {
+		return
+	}
+	enc = _enc
+	return
+}
+
+func NewAudioEncoderByName(name string) (enc *AudioEncoder, err error) {
+	_enc := &AudioEncoder{}
+
+	codec := C.avcodec_find_encoder_by_name(C.CString(name))
+	if codec == nil || C.avcodec_get_type(codec.id) != C.AVMEDIA_TYPE_AUDIO {
+		err = fmt.Errorf("ffmpeg: cannot find audio encoder name=%s", name)
+		return
+	}
+
+	if _enc.ff, err = newFFCtxByCodec(codec); err != nil {
+		return
+	}
+	enc = _enc
+	return
+}
+
+func NewAudioDecoder(codec av.AudioCodecData) (dec *AudioDecoder, err error) {
+	_dec := &AudioDecoder{}
+	var id uint32
+
+	switch codec.Type() {
+	case av.AAC:
+		if aaccodec, ok := codec.(aacparser.CodecData); ok {
+			_dec.Extradata = aaccodec.MPEG4AudioConfigBytes()
+			id = C.AV_CODEC_ID_AAC
+		} else {
+			err = fmt.Errorf("ffmpeg: aac CodecData must be aacparser.CodecData")
+			return
+		}
+
+	case av.SPEEX:
+		id = C.AV_CODEC_ID_SPEEX
+
+	case av.PCM_MULAW:
+		id = C.AV_CODEC_ID_PCM_MULAW
+
+	case av.PCM_ALAW:
+		id = C.AV_CODEC_ID_PCM_ALAW
+
+	default:
+		if ffcodec, ok := codec.(audioCodecData); ok {
+			_dec.Extradata = ffcodec.extradata
+			id = ffcodec.codecId
+		} else {
+			err = fmt.Errorf("ffmpeg: invalid CodecData for ffmpeg to decode")
+			return
+		}
+	}
+
+	c := C.avcodec_find_decoder(id)
+	if c == nil || C.avcodec_get_type(c.id) != C.AVMEDIA_TYPE_AUDIO {
+		err = fmt.Errorf("ffmpeg: cannot find audio decoder id=%d", id)
+		return
+	}
+
+	if _dec.ff, err = newFFCtxByCodec(c); err != nil {
+		return
+	}
+
+	_dec.SampleFormat = codec.SampleFormat()
+	_dec.SampleRate = codec.SampleRate()
+	_dec.ChannelLayout = codec.ChannelLayout()
+	if err = _dec.Setup(); err != nil {
+		return
+	}
+
+	dec = _dec
+	return
+}
+
+type audioCodecData struct {
+	codecId       uint32
+	sampleFormat  av.SampleFormat
+	channelLayout av.ChannelLayout
+	sampleRate    int
+	extradata     []byte
+}
+
+func (self audioCodecData) Type() av.CodecType {
+	return av.MakeAudioCodecType(self.codecId)
+}
+
+func (self audioCodecData) SampleRate() int {
+	return self.sampleRate
+}
+
+func (self audioCodecData) SampleFormat() av.SampleFormat {
+	return self.sampleFormat
+}
+
+func (self audioCodecData) ChannelLayout() av.ChannelLayout {
+	return self.channelLayout
+}
+
+func (self audioCodecData) PacketDuration(data []byte) (dur time.Duration, err error) {
+	// TODO: implement it: ffmpeg get_audio_frame_duration
+	err = fmt.Errorf("ffmpeg: cannot get packet duration")
+	return
+}
+
+func AudioCodecHandler(h *avutil.RegisterHandler) {
+	h.AudioDecoder = func(codec av.AudioCodecData) (av.AudioDecoder, error) {
+		if dec, err := NewAudioDecoder(codec); err != nil {
+			return nil, nil
+		} else {
+			return dec, err
+		}
+	}
+
+	h.AudioEncoder = func(typ av.CodecType) (av.AudioEncoder, error) {
+		if enc, err := NewAudioEncoderByCodecType(typ); err != nil {
+			return nil, nil
+		} else {
+			return enc, err
+		}
+	}
+}

cgo/ffmpeg/ffmpeg.go

@@ -0,0 +1,74 @@
+package ffmpeg
+
+/*
+#cgo LDFLAGS: -lavformat -lavutil -lavcodec -lavresample -lswscale
+#include "ffmpeg.h"
+void ffinit() {
+	av_register_all();
+}
+*/
+import "C"
+import (
+	"runtime"
+	"unsafe"
+)
+
+const (
+	QUIET = int(C.AV_LOG_QUIET)
+	PANIC = int(C.AV_LOG_PANIC)
+	FATAL = int(C.AV_LOG_FATAL)
+	ERROR = int(C.AV_LOG_ERROR)
+	WARNING = int(C.AV_LOG_WARNING)
+	INFO = int(C.AV_LOG_INFO)
+	VERBOSE = int(C.AV_LOG_VERBOSE)
+	DEBUG = int(C.AV_LOG_DEBUG)
+	TRACE = int(C.AV_LOG_TRACE)
+)
+
+func HasEncoder(name string) bool {
+	return C.avcodec_find_encoder_by_name(C.CString(name)) != nil
+}
+
+func HasDecoder(name string) bool {
+	return C.avcodec_find_decoder_by_name(C.CString(name)) != nil
+}
+
+//func EncodersList() []string
+//func DecodersList() []string
+
+func SetLogLevel(level int) {
+	C.av_log_set_level(C.int(level))
+}
+
+func init() {
+	C.ffinit()
+}
+
+type ffctx struct {
+	ff C.FFCtx
+}
+
+func newFFCtxByCodec(codec *C.AVCodec) (ff *ffctx, err error) {
+	ff = &ffctx{}
+	ff.ff.codec = codec
+	ff.ff.codecCtx = C.avcodec_alloc_context3(codec)
+	ff.ff.profile = C.FF_PROFILE_UNKNOWN
+	runtime.SetFinalizer(ff, freeFFCtx)
+	return
+}
+
+func freeFFCtx(self *ffctx) {
+	ff := &self.ff
+	if ff.frame != nil {
+		C.av_frame_free(&ff.frame)
+	}
+	if ff.codecCtx != nil {
+		C.avcodec_close(ff.codecCtx)
+		C.av_free(unsafe.Pointer(ff.codecCtx))
+		ff.codecCtx = nil
+	}
+	if ff.options != nil {
+		C.av_dict_free(&ff.options)
+	}
+}
+

cgo/ffmpeg/ffmpeg.h

@@ -0,0 +1,25 @@
+
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libavutil/avutil.h>
+#include <libavresample/avresample.h>
+#include <libavutil/opt.h>
+#include <string.h>
+#include <libswscale/swscale.h>
+
+typedef struct {
+	AVCodec *codec;
+	AVCodecContext *codecCtx;
+	AVFrame *frame;
+	AVDictionary *options;
+	int profile;
+} FFCtx;
+
+static inline int avcodec_profile_name_to_int(AVCodec *codec, const char *name) {
+	const AVProfile *p;
+	for (p = codec->profiles; p != NULL && p->profile != FF_PROFILE_UNKNOWN; p++)
+		if (!strcasecmp(p->name, name))
+			return p->profile;
+	return FF_PROFILE_UNKNOWN;
+}
+

cgo/ffmpeg/video.go

@@ -0,0 +1,124 @@
+package ffmpeg
+
+/*
+#include "ffmpeg.h"
+int wrap_avcodec_decode_video2(AVCodecContext *ctx, AVFrame *frame, void *data, int size, int *got) {
+	struct AVPacket pkt = {.data = data, .size = size};
+	return avcodec_decode_video2(ctx, frame, got, &pkt);
+}
+*/
+import "C"
+import (
+	"fmt"
+	"image"
+	"reflect"
+	"runtime"
+	"unsafe"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/codec/h264parser"
+)
+
+type VideoDecoder struct {
+	ff        *ffctx
+	Extradata []byte
+}
+
+func (self *VideoDecoder) Setup() (err error) {
+	ff := &self.ff.ff
+	if len(self.Extradata) > 0 {
+		ff.codecCtx.extradata = (*C.uint8_t)(unsafe.Pointer(&self.Extradata[0]))
+		ff.codecCtx.extradata_size = C.int(len(self.Extradata))
+	}
+	if C.avcodec_open2(ff.codecCtx, ff.codec, nil) != 0 {
+		err = fmt.Errorf("ffmpeg: decoder: avcodec_open2 failed")
+		return
+	}
+	return
+}
+
+func fromCPtr(buf unsafe.Pointer, size int) (ret []uint8) {
+	hdr := (*reflect.SliceHeader)((unsafe.Pointer(&ret)))
+	hdr.Cap = size
+	hdr.Len = size
+	hdr.Data = uintptr(buf)
+	return
+}
+
+type VideoFrame struct {
+	Image image.YCbCr
+	frame *C.AVFrame
+}
+
+func (self *VideoFrame) Free() {
+	self.Image = image.YCbCr{}
+	C.av_frame_free(&self.frame)
+}
+
+func freeVideoFrame(self *VideoFrame) {
+	self.Free()
+}
+
+func (self *VideoDecoder) Decode(pkt []byte) (img *VideoFrame, err error) {
+	ff := &self.ff.ff
+
+	cgotimg := C.int(0)
+	frame := C.av_frame_alloc()
+	cerr := C.wrap_avcodec_decode_video2(ff.codecCtx, frame, unsafe.Pointer(&pkt[0]), C.int(len(pkt)), &cgotimg)
+	if cerr < C.int(0) {
+		err = fmt.Errorf("ffmpeg: avcodec_decode_video2 failed: %d", cerr)
+		return
+	}
+
+	if cgotimg != C.int(0) {
+		w := int(frame.width)
+		h := int(frame.height)
+		ys := int(frame.linesize[0])
+		cs := int(frame.linesize[1])
+
+		img = &VideoFrame{Image: image.YCbCr{
+			Y:              fromCPtr(unsafe.Pointer(frame.data[0]), ys*h),
+			Cb:             fromCPtr(unsafe.Pointer(frame.data[1]), cs*h/2),
+			Cr:             fromCPtr(unsafe.Pointer(frame.data[2]), cs*h/2),
+			YStride:        ys,
+			CStride:        cs,
+			SubsampleRatio: image.YCbCrSubsampleRatio420,
+			Rect:           image.Rect(0, 0, w, h),
+		}, frame: frame}
+		runtime.SetFinalizer(img, freeVideoFrame)
+	}
+
+	return
+}
+
+func NewVideoDecoder(stream av.CodecData) (dec *VideoDecoder, err error) {
+	_dec := &VideoDecoder{}
+	var id uint32
+
+	switch stream.Type() {
+	case av.H264:
+		h264 := stream.(h264parser.CodecData)
+		_dec.Extradata = h264.AVCDecoderConfRecordBytes()
+		id = C.AV_CODEC_ID_H264
+
+	default:
+		err = fmt.Errorf("ffmpeg: NewVideoDecoder codec=%v unsupported", stream.Type())
+		return
+	}
+
+	c := C.avcodec_find_decoder(id)
+	if c == nil || C.avcodec_get_type(id) != C.AVMEDIA_TYPE_VIDEO {
+		err = fmt.Errorf("ffmpeg: cannot find video decoder codecId=%d", id)
+		return
+	}
+
+	if _dec.ff, err = newFFCtxByCodec(c); err != nil {
+		return
+	}
+	if err = _dec.Setup(); err != nil {
+		return
+	}
+
+	dec = _dec
+	return
+}

codec/aacparser/parser.go

@@ -0,0 +1,311 @@
+package aacparser
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/utils/bits"
+)
+
+// copied from libavcodec/mpeg4audio.h
+const (
+	AOT_AAC_MAIN        = 1 + iota  ///< Y                       Main
+	AOT_AAC_LC                      ///< Y                       Low Complexity
+	AOT_AAC_SSR                     ///< N (code in SoC repo)    Scalable Sample Rate
+	AOT_AAC_LTP                     ///< Y                       Long Term Prediction
+	AOT_SBR                         ///< Y                       Spectral Band Replication
+	AOT_AAC_SCALABLE                ///< N                       Scalable
+	AOT_TWINVQ                      ///< N                       Twin Vector Quantizer
+	AOT_CELP                        ///< N                       Code Excited Linear Prediction
+	AOT_HVXC                        ///< N                       Harmonic Vector eXcitation Coding
+	AOT_TTSI            = 12 + iota ///< N                       Text-To-Speech Interface
+	AOT_MAINSYNTH                   ///< N                       Main Synthesis
+	AOT_WAVESYNTH                   ///< N                       Wavetable Synthesis
+	AOT_MIDI                        ///< N                       General MIDI
+	AOT_SAFX                        ///< N                       Algorithmic Synthesis and Audio Effects
+	AOT_ER_AAC_LC                   ///< N                       Error Resilient Low Complexity
+	AOT_ER_AAC_LTP      = 19 + iota ///< N                       Error Resilient Long Term Prediction
+	AOT_ER_AAC_SCALABLE             ///< N                       Error Resilient Scalable
+	AOT_ER_TWINVQ                   ///< N                       Error Resilient Twin Vector Quantizer
+	AOT_ER_BSAC                     ///< N                       Error Resilient Bit-Sliced Arithmetic Coding
+	AOT_ER_AAC_LD                   ///< N                       Error Resilient Low Delay
+	AOT_ER_CELP                     ///< N                       Error Resilient Code Excited Linear Prediction
+	AOT_ER_HVXC                     ///< N                       Error Resilient Harmonic Vector eXcitation Coding
+	AOT_ER_HILN                     ///< N                       Error Resilient Harmonic and Individual Lines plus Noise
+	AOT_ER_PARAM                    ///< N                       Error Resilient Parametric
+	AOT_SSC                         ///< N                       SinuSoidal Coding
+	AOT_PS                          ///< N                       Parametric Stereo
+	AOT_SURROUND                    ///< N                       MPEG Surround
+	AOT_ESCAPE                      ///< Y                       Escape Value
+	AOT_L1                          ///< Y                       Layer 1
+	AOT_L2                          ///< Y                       Layer 2
+	AOT_L3                          ///< Y                       Layer 3
+	AOT_DST                         ///< N                       Direct Stream Transfer
+	AOT_ALS                         ///< Y                       Audio LosslesS
+	AOT_SLS                         ///< N                       Scalable LosslesS
+	AOT_SLS_NON_CORE                ///< N                       Scalable LosslesS (non core)
+	AOT_ER_AAC_ELD                  ///< N                       Error Resilient Enhanced Low Delay
+	AOT_SMR_SIMPLE                  ///< N                       Symbolic Music Representation Simple
+	AOT_SMR_MAIN                    ///< N                       Symbolic Music Representation Main
+	AOT_USAC_NOSBR                  ///< N                       Unified Speech and Audio Coding (no SBR)
+	AOT_SAOC                        ///< N                       Spatial Audio Object Coding
+	AOT_LD_SURROUND                 ///< N                       Low Delay MPEG Surround
+	AOT_USAC                        ///< N                       Unified Speech and Audio Coding
+)
+
+type MPEG4AudioConfig struct {
+	SampleRate      int
+	ChannelLayout   av.ChannelLayout
+	ObjectType      uint
+	SampleRateIndex uint
+	ChannelConfig   uint
+}
+
+var sampleRateTable = []int{
+	96000, 88200, 64000, 48000, 44100, 32000,
+	24000, 22050, 16000, 12000, 11025, 8000, 7350,
+}
+
+/*
+These are the channel configurations:
+0: Defined in AOT Specifc Config
+1: 1 channel: front-center
+2: 2 channels: front-left, front-right
+3: 3 channels: front-center, front-left, front-right
+4: 4 channels: front-center, front-left, front-right, back-center
+5: 5 channels: front-center, front-left, front-right, back-left, back-right
+6: 6 channels: front-center, front-left, front-right, back-left, back-right, LFE-channel
+7: 8 channels: front-center, front-left, front-right, side-left, side-right, back-left, back-right, LFE-channel
+8-15: Reserved
+*/
+var chanConfigTable = []av.ChannelLayout{
+	0,
+	av.CH_FRONT_CENTER,
+	av.CH_FRONT_LEFT | av.CH_FRONT_RIGHT,
+	av.CH_FRONT_CENTER | av.CH_FRONT_LEFT | av.CH_FRONT_RIGHT,
+	av.CH_FRONT_CENTER | av.CH_FRONT_LEFT | av.CH_FRONT_RIGHT | av.CH_BACK_CENTER,
+	av.CH_FRONT_CENTER | av.CH_FRONT_LEFT | av.CH_FRONT_RIGHT | av.CH_BACK_LEFT | av.CH_BACK_RIGHT,
+	av.CH_FRONT_CENTER | av.CH_FRONT_LEFT | av.CH_FRONT_RIGHT | av.CH_BACK_LEFT | av.CH_BACK_RIGHT | av.CH_LOW_FREQ,
+	av.CH_FRONT_CENTER | av.CH_FRONT_LEFT | av.CH_FRONT_RIGHT | av.CH_SIDE_LEFT | av.CH_SIDE_RIGHT | av.CH_BACK_LEFT | av.CH_BACK_RIGHT | av.CH_LOW_FREQ,
+}
+
+func ParseADTSHeader(frame []byte) (config MPEG4AudioConfig, hdrlen int, framelen int, samples int, err error) {
+	if frame[0] != 0xff || frame[1]&0xf6 != 0xf0 {
+		err = fmt.Errorf("aacparser: not adts header")
+		return
+	}
+	config.ObjectType = uint(frame[2]>>6) + 1
+	config.SampleRateIndex = uint(frame[2] >> 2 & 0xf)
+	config.ChannelConfig = uint(frame[2]<<2&0x4 | frame[3]>>6&0x3)
+	if config.ChannelConfig == uint(0) {
+		err = fmt.Errorf("aacparser: adts channel count invalid")
+		return
+	}
+	(&config).Complete()
+	framelen = int(frame[3]&0x3)<<11 | int(frame[4])<<3 | int(frame[5]>>5)
+	samples = (int(frame[6]&0x3) + 1) * 1024
+	hdrlen = 7
+	if frame[1]&0x1 == 0 {
+		hdrlen = 9
+	}
+	if framelen < hdrlen {
+		err = fmt.Errorf("aacparser: adts framelen < hdrlen")
+		return
+	}
+	return
+}
+
+const ADTSHeaderLength = 7
+
+func FillADTSHeader(header []byte, config MPEG4AudioConfig, samples int, payloadLength int) {
+	payloadLength += 7
+	//AAAAAAAA AAAABCCD EEFFFFGH HHIJKLMM MMMMMMMM MMMOOOOO OOOOOOPP (QQQQQQQQ QQQQQQQQ)
+	header[0] = 0xff
+	header[1] = 0xf1
+	header[2] = 0x50
+	header[3] = 0x80
+	header[4] = 0x43
+	header[5] = 0xff
+	header[6] = 0xcd
+	//config.ObjectType = uint(frames[2]>>6)+1
+	//config.SampleRateIndex = uint(frames[2]>>2&0xf)
+	//config.ChannelConfig = uint(frames[2]<<2&0x4|frames[3]>>6&0x3)
+	header[2] = (byte(config.ObjectType-1)&0x3)<<6 | (byte(config.SampleRateIndex)&0xf)<<2 | byte(config.ChannelConfig>>2)&0x1
+	header[3] = header[3]&0x3f | byte(config.ChannelConfig&0x3)<<6
+	header[3] = header[3]&0xfc | byte(payloadLength>>11)&0x3
+	header[4] = byte(payloadLength >> 3)
+	header[5] = header[5]&0x1f | (byte(payloadLength)&0x7)<<5
+	header[6] = header[6]&0xfc | byte(samples/1024-1)
+	return
+}
+
+func readObjectType(r *bits.Reader) (objectType uint, err error) {
+	if objectType, err = r.ReadBits(5); err != nil {
+		return
+	}
+	if objectType == AOT_ESCAPE {
+		var i uint
+		if i, err = r.ReadBits(6); err != nil {
+			return
+		}
+		objectType = 32 + i
+	}
+	return
+}
+
+func writeObjectType(w *bits.Writer, objectType uint) (err error) {
+	if objectType >= 32 {
+		if err = w.WriteBits(AOT_ESCAPE, 5); err != nil {
+			return
+		}
+		if err = w.WriteBits(objectType-32, 6); err != nil {
+			return
+		}
+	} else {
+		if err = w.WriteBits(objectType, 5); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func readSampleRateIndex(r *bits.Reader) (index uint, err error) {
+	if index, err = r.ReadBits(4); err != nil {
+		return
+	}
+	if index == 0xf {
+		if index, err = r.ReadBits(24); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func writeSampleRateIndex(w *bits.Writer, index uint) (err error) {
+	if index >= 0xf {
+		if err = w.WriteBits(0xf, 4); err != nil {
+			return
+		}
+		if err = w.WriteBits(index, 24); err != nil {
+			return
+		}
+	} else {
+		if err = w.WriteBits(index, 4); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (self MPEG4AudioConfig) IsValid() bool {
+	return self.ObjectType > 0
+}
+
+func (self *MPEG4AudioConfig) Complete() {
+	if int(self.SampleRateIndex) < len(sampleRateTable) {
+		self.SampleRate = sampleRateTable[self.SampleRateIndex]
+	}
+	if int(self.ChannelConfig) < len(chanConfigTable) {
+		self.ChannelLayout = chanConfigTable[self.ChannelConfig]
+	}
+	return
+}
+
+func ParseMPEG4AudioConfigBytes(data []byte) (config MPEG4AudioConfig, err error) {
+	// copied from libavcodec/mpeg4audio.c avpriv_mpeg4audio_get_config()
+	r := bytes.NewReader(data)
+	br := &bits.Reader{R: r}
+	if config.ObjectType, err = readObjectType(br); err != nil {
+		return
+	}
+	if config.SampleRateIndex, err = readSampleRateIndex(br); err != nil {
+		return
+	}
+	if config.ChannelConfig, err = br.ReadBits(4); err != nil {
+		return
+	}
+	(&config).Complete()
+	return
+}
+
+func WriteMPEG4AudioConfig(w io.Writer, config MPEG4AudioConfig) (err error) {
+	bw := &bits.Writer{W: w}
+	if err = writeObjectType(bw, config.ObjectType); err != nil {
+		return
+	}
+
+	if config.SampleRateIndex == 0 {
+		for i, rate := range sampleRateTable {
+			if rate == config.SampleRate {
+				config.SampleRateIndex = uint(i)
+			}
+		}
+	}
+	if err = writeSampleRateIndex(bw, config.SampleRateIndex); err != nil {
+		return
+	}
+
+	if config.ChannelConfig == 0 {
+		for i, layout := range chanConfigTable {
+			if layout == config.ChannelLayout {
+				config.ChannelConfig = uint(i)
+			}
+		}
+	}
+	if err = bw.WriteBits(config.ChannelConfig, 4); err != nil {
+		return
+	}
+
+	if err = bw.FlushBits(); err != nil {
+		return
+	}
+	return
+}
+
+type CodecData struct {
+	ConfigBytes []byte
+	Config      MPEG4AudioConfig
+}
+
+func (self CodecData) Type() av.CodecType {
+	return av.AAC
+}
+
+func (self CodecData) MPEG4AudioConfigBytes() []byte {
+	return self.ConfigBytes
+}
+
+func (self CodecData) ChannelLayout() av.ChannelLayout {
+	return self.Config.ChannelLayout
+}
+
+func (self CodecData) SampleRate() int {
+	return self.Config.SampleRate
+}
+
+func (self CodecData) SampleFormat() av.SampleFormat {
+	return av.FLTP
+}
+
+func (self CodecData) PacketDuration(data []byte) (dur time.Duration, err error) {
+	dur = time.Duration(1024) * time.Second / time.Duration(self.Config.SampleRate)
+	return
+}
+
+func NewCodecDataFromMPEG4AudioConfig(config MPEG4AudioConfig) (self CodecData, err error) {
+	b := &bytes.Buffer{}
+	WriteMPEG4AudioConfig(b, config)
+	return NewCodecDataFromMPEG4AudioConfigBytes(b.Bytes())
+}
+
+func NewCodecDataFromMPEG4AudioConfigBytes(config []byte) (self CodecData, err error) {
+	self.ConfigBytes = config
+	if self.Config, err = ParseMPEG4AudioConfigBytes(config); err != nil {
+		err = fmt.Errorf("aacparser: parse MPEG4AudioConfig failed(%s)", err)
+		return
+	}
+	return
+}

codec/codec.go

@@ -0,0 +1,64 @@
+package codec
+
+import (
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/codec/fake"
+)
+
+type PCMUCodecData struct {
+	typ av.CodecType
+}
+
+func (self PCMUCodecData) Type() av.CodecType {
+	return self.typ
+}
+
+func (self PCMUCodecData) SampleRate() int {
+	return 8000
+}
+
+func (self PCMUCodecData) ChannelLayout() av.ChannelLayout {
+	return av.CH_MONO
+}
+
+func (self PCMUCodecData) SampleFormat() av.SampleFormat {
+	return av.S16
+}
+
+func (self PCMUCodecData) PacketDuration(data []byte) (time.Duration, error) {
+	return time.Duration(len(data)) * time.Second / time.Duration(8000), nil
+}
+
+func NewPCMMulawCodecData() av.AudioCodecData {
+	return PCMUCodecData{
+		typ: av.PCM_MULAW,
+	}
+}
+
+func NewPCMAlawCodecData() av.AudioCodecData {
+	return PCMUCodecData{
+		typ: av.PCM_ALAW,
+	}
+}
+
+type SpeexCodecData struct {
+	fake.CodecData
+}
+
+func (self SpeexCodecData) PacketDuration(data []byte) (time.Duration, error) {
+	// libavcodec/libspeexdec.c
+	// samples = samplerate/50
+	// duration = 0.02s
+	return time.Millisecond * 20, nil
+}
+
+func NewSpeexCodecData(sr int, cl av.ChannelLayout) SpeexCodecData {
+	codec := SpeexCodecData{}
+	codec.CodecType_ = av.SPEEX
+	codec.SampleFormat_ = av.S16
+	codec.SampleRate_ = sr
+	codec.ChannelLayout_ = cl
+	return codec
+}

codec/fake/fake.go

@@ -0,0 +1,28 @@
+package fake
+
+import (
+	"github.com/deepch/vdk/av"
+)
+
+type CodecData struct {
+	CodecType_     av.CodecType
+	SampleRate_    int
+	SampleFormat_  av.SampleFormat
+	ChannelLayout_ av.ChannelLayout
+}
+
+func (self CodecData) Type() av.CodecType {
+	return self.CodecType_
+}
+
+func (self CodecData) SampleFormat() av.SampleFormat {
+	return self.SampleFormat_
+}
+
+func (self CodecData) ChannelLayout() av.ChannelLayout {
+	return self.ChannelLayout_
+}
+
+func (self CodecData) SampleRate() int {
+	return self.SampleRate_
+}

codec/h264parser/parser.go

@@ -0,0 +1,742 @@
+package h264parser
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/utils/bits"
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+const (
+	NALU_SEI = 6
+	NALU_PPS = 7
+	NALU_SPS = 8
+	NALU_AUD = 9
+)
+
+func IsDataNALU(b []byte) bool {
+	typ := b[0] & 0x1f
+	return typ >= 1 && typ <= 5
+}
+
+/*
+From: http://stackoverflow.com/questions/24884827/possible-locations-for-sequence-picture-parameter-sets-for-h-264-stream
+
+First off, it's important to understand that there is no single standard H.264 elementary bitstream format. The specification document does contain an Annex, specifically Annex B, that describes one possible format, but it is not an actual requirement. The standard specifies how video is encoded into individual packets. How these packets are stored and transmitted is left open to the integrator.
+
+1. Annex B
+Network Abstraction Layer Units
+The packets are called Network Abstraction Layer Units. Often abbreviated NALU (or sometimes just NAL) each packet can be individually parsed and processed. The first byte of each NALU contains the NALU type, specifically bits 3 through 7. (bit 0 is always off, and bits 1-2 indicate whether a NALU is referenced by another NALU).
+
+There are 19 different NALU types defined separated into two categories, VCL and non-VCL:
+
+VCL, or Video Coding Layer packets contain the actual visual information.
+Non-VCLs contain metadata that may or may not be required to decode the video.
+A single NALU, or even a VCL NALU is NOT the same thing as a frame. A frame can be ‘sliced’ into several NALUs. Just like you can slice a pizza. One or more slices are then virtually grouped into a Access Units (AU) that contain one frame. Slicing does come at a slight quality cost, so it is not often used.
+
+Below is a table of all defined NALUs.
+
+0      Unspecified                                                    non-VCL
+1      Coded slice of a non-IDR picture                               VCL
+2      Coded slice data partition A                                   VCL
+3      Coded slice data partition B                                   VCL
+4      Coded slice data partition C                                   VCL
+5      Coded slice of an IDR picture                                  VCL
+6      Supplemental enhancement information (SEI)                     non-VCL
+7      Sequence parameter set                                         non-VCL
+8      Picture parameter set                                          non-VCL
+9      Access unit delimiter                                          non-VCL
+10     End of sequence                                                non-VCL
+11     End of stream                                                  non-VCL
+12     Filler data                                                    non-VCL
+13     Sequence parameter set extension                               non-VCL
+14     Prefix NAL unit                                                non-VCL
+15     Subset sequence parameter set                                  non-VCL
+16     Depth parameter set                                            non-VCL
+17..18 Reserved                                                       non-VCL
+19     Coded slice of an auxiliary coded picture without partitioning non-VCL
+20     Coded slice extension                                          non-VCL
+21     Coded slice extension for depth view components                non-VCL
+22..23 Reserved                                                       non-VCL
+24..31 Unspecified                                                    non-VCL
+There are a couple of NALU types where having knowledge of may be helpful later.
+
+Sequence Parameter Set (SPS). This non-VCL NALU contains information required to configure the decoder such as profile, level, resolution, frame rate.
+Picture Parameter Set (PPS). Similar to the SPS, this non-VCL contains information on entropy coding mode, slice groups, motion prediction and deblocking filters.
+Instantaneous Decoder Refresh (IDR). This VCL NALU is a self contained image slice. That is, an IDR can be decoded and displayed without referencing any other NALU save SPS and PPS.
+Access Unit Delimiter (AUD). An AUD is an optional NALU that can be use to delimit frames in an elementary stream. It is not required (unless otherwise stated by the container/protocol, like TS), and is often not included in order to save space, but it can be useful to finds the start of a frame without having to fully parse each NALU.
+NALU Start Codes
+A NALU does not contain is its size. Therefore simply concatenating the NALUs to create a stream will not work because you will not know where one stops and the next begins.
+
+The Annex B specification solves this by requiring ‘Start Codes’ to precede each NALU. A start code is 2 or 3 0x00 bytes followed with a 0x01 byte. e.g. 0x000001 or 0x00000001.
+
+The 4 byte variation is useful for transmission over a serial connection as it is trivial to byte align the stream by looking for 31 zero bits followed by a one. If the next bit is 0 (because every NALU starts with a 0 bit), it is the start of a NALU. The 4 byte variation is usually only used for signaling random access points in the stream such as a SPS PPS AUD and IDR Where as the 3 byte variation is used everywhere else to save space.
+
+Emulation Prevention Bytes
+Start codes work because the four byte sequences 0x000000, 0x000001, 0x000002 and 0x000003 are illegal within a non-RBSP NALU. So when creating a NALU, care is taken to escape these values that could otherwise be confused with a start code. This is accomplished by inserting an ‘Emulation Prevention’ byte 0x03, so that 0x000001 becomes 0x00000301.
+
+When decoding, it is important to look for and ignore emulation prevention bytes. Because emulation prevention bytes can occur almost anywhere within a NALU, it is often more convenient in documentation to assume they have already been removed. A representation without emulation prevention bytes is called Raw Byte Sequence Payload (RBSP).
+
+Example
+Let's look at a complete example.
+
+0x0000 | 00 00 00 01 67 64 00 0A AC 72 84 44 26 84 00 00
+0x0010 | 03 00 04 00 00 03 00 CA 3C 48 96 11 80 00 00 00
+0x0020 | 01 68 E8 43 8F 13 21 30 00 00 01 65 88 81 00 05
+0x0030 | 4E 7F 87 DF 61 A5 8B 95 EE A4 E9 38 B7 6A 30 6A
+0x0040 | 71 B9 55 60 0B 76 2E B5 0E E4 80 59 27 B8 67 A9
+0x0050 | 63 37 5E 82 20 55 FB E4 6A E9 37 35 72 E2 22 91
+0x0060 | 9E 4D FF 60 86 CE 7E 42 B7 95 CE 2A E1 26 BE 87
+0x0070 | 73 84 26 BA 16 36 F4 E6 9F 17 DA D8 64 75 54 B1
+0x0080 | F3 45 0C 0B 3C 74 B3 9D BC EB 53 73 87 C3 0E 62
+0x0090 | 47 48 62 CA 59 EB 86 3F 3A FA 86 B5 BF A8 6D 06
+0x00A0 | 16 50 82 C4 CE 62 9E 4E E6 4C C7 30 3E DE A1 0B
+0x00B0 | D8 83 0B B6 B8 28 BC A9 EB 77 43 FC 7A 17 94 85
+0x00C0 | 21 CA 37 6B 30 95 B5 46 77 30 60 B7 12 D6 8C C5
+0x00D0 | 54 85 29 D8 69 A9 6F 12 4E 71 DF E3 E2 B1 6B 6B
+0x00E0 | BF 9F FB 2E 57 30 A9 69 76 C4 46 A2 DF FA 91 D9
+0x00F0 | 50 74 55 1D 49 04 5A 1C D6 86 68 7C B6 61 48 6C
+0x0100 | 96 E6 12 4C 27 AD BA C7 51 99 8E D0 F0 ED 8E F6
+0x0110 | 65 79 79 A6 12 A1 95 DB C8 AE E3 B6 35 E6 8D BC
+0x0120 | 48 A3 7F AF 4A 28 8A 53 E2 7E 68 08 9F 67 77 98
+0x0130 | 52 DB 50 84 D6 5E 25 E1 4A 99 58 34 C7 11 D6 43
+0x0140 | FF C4 FD 9A 44 16 D1 B2 FB 02 DB A1 89 69 34 C2
+0x0150 | 32 55 98 F9 9B B2 31 3F 49 59 0C 06 8C DB A5 B2
+0x0160 | 9D 7E 12 2F D0 87 94 44 E4 0A 76 EF 99 2D 91 18
+0x0170 | 39 50 3B 29 3B F5 2C 97 73 48 91 83 B0 A6 F3 4B
+0x0180 | 70 2F 1C 8F 3B 78 23 C6 AA 86 46 43 1D D7 2A 23
+0x0190 | 5E 2C D9 48 0A F5 F5 2C D1 FB 3F F0 4B 78 37 E9
+0x01A0 | 45 DD 72 CF 80 35 C3 95 07 F3 D9 06 E5 4A 58 76
+0x01B0 | 03 6C 81 20 62 45 65 44 73 BC FE C1 9F 31 E5 DB
+0x01C0 | 89 5C 6B 79 D8 68 90 D7 26 A8 A1 88 86 81 DC 9A
+0x01D0 | 4F 40 A5 23 C7 DE BE 6F 76 AB 79 16 51 21 67 83
+0x01E0 | 2E F3 D6 27 1A 42 C2 94 D1 5D 6C DB 4A 7A E2 CB
+0x01F0 | 0B B0 68 0B BE 19 59 00 50 FC C0 BD 9D F5 F5 F8
+0x0200 | A8 17 19 D6 B3 E9 74 BA 50 E5 2C 45 7B F9 93 EA
+0x0210 | 5A F9 A9 30 B1 6F 5B 36 24 1E 8D 55 57 F4 CC 67
+0x0220 | B2 65 6A A9 36 26 D0 06 B8 E2 E3 73 8B D1 C0 1C
+0x0230 | 52 15 CA B5 AC 60 3E 36 42 F1 2C BD 99 77 AB A8
+0x0240 | A9 A4 8E 9C 8B 84 DE 73 F0 91 29 97 AE DB AF D6
+0x0250 | F8 5E 9B 86 B3 B3 03 B3 AC 75 6F A6 11 69 2F 3D
+0x0260 | 3A CE FA 53 86 60 95 6C BB C5 4E F3
+
+This is a complete AU containing 3 NALUs. As you can see, we begin with a Start code followed by an SPS (SPS starts with 67). Within the SPS, you will see two Emulation Prevention bytes. Without these bytes the illegal sequence 0x000000 would occur at these positions. Next you will see a start code followed by a PPS (PPS starts with 68) and one final start code followed by an IDR slice. This is a complete H.264 stream. If you type these values into a hex editor and save the file with a .264 extension, you will be able to convert it to this image:
+
+Lena
+
+Annex B is commonly used in live and streaming formats such as transport streams, over the air broadcasts, and DVDs. In these formats it is common to repeat the SPS and PPS periodically, usually preceding every IDR thus creating a random access point for the decoder. This enables the ability to join a stream already in progress.
+
+2. AVCC
+The other common method of storing an H.264 stream is the AVCC format. In this format, each NALU is preceded with its length (in big endian format). This method is easier to parse, but you lose the byte alignment features of Annex B. Just to complicate things, the length may be encoded using 1, 2 or 4 bytes. This value is stored in a header object. This header is often called ‘extradata’ or ‘sequence header’. Its basic format is as follows:
+
+bits
+8   version ( always 0x01 )
+8   avc profile ( sps[0][1] )
+8   avc compatibility ( sps[0][2] )
+8   avc level ( sps[0][3] )
+6   reserved ( all bits on )
+2   NALULengthSizeMinusOne
+3   reserved ( all bits on )
+5   number of SPS NALUs (usually 1)
+repeated once per SPS:
+  16         SPS size
+	variable   SPS NALU data
+8   number of PPS NALUs (usually 1)
+repeated once per PPS
+  16         PPS size
+  variable   PPS NALU data
+
+Using the same example above, the AVCC extradata will look like this:
+
+0x0000 | 01 64 00 0A FF E1 00 19 67 64 00 0A AC 72 84 44
+0x0010 | 26 84 00 00 03 00 04 00 00 03 00 CA 3C 48 96 11
+0x0020 | 80 01 00 07 68 E8 43 8F 13 21 30
+
+You will notice SPS and PPS is now stored out of band. That is, separate from the elementary stream data. Storage and transmission of this data is the job of the file container, and beyond the scope of this document. Notice that even though we are not using start codes, emulation prevention bytes are still inserted.
+
+Additionally, there is a new variable called NALULengthSizeMinusOne. This confusingly named variable tells us how many bytes to use to store the length of each NALU. So, if NALULengthSizeMinusOne is set to 0, then each NALU is preceded with a single byte indicating its length. Using a single byte to store the size, the max size of a NALU is 255 bytes. That is obviously pretty small. Way too small for an entire key frame. Using 2 bytes gives us 64k per NALU. It would work in our example, but is still a pretty low limit. 3 bytes would be perfect, but for some reason is not universally supported. Therefore, 4 bytes is by far the most common, and it is what we used here:
+
+0x0000 | 00 00 02 41 65 88 81 00 05 4E 7F 87 DF 61 A5 8B
+0x0010 | 95 EE A4 E9 38 B7 6A 30 6A 71 B9 55 60 0B 76 2E
+0x0020 | B5 0E E4 80 59 27 B8 67 A9 63 37 5E 82 20 55 FB
+0x0030 | E4 6A E9 37 35 72 E2 22 91 9E 4D FF 60 86 CE 7E
+0x0040 | 42 B7 95 CE 2A E1 26 BE 87 73 84 26 BA 16 36 F4
+0x0050 | E6 9F 17 DA D8 64 75 54 B1 F3 45 0C 0B 3C 74 B3
+0x0060 | 9D BC EB 53 73 87 C3 0E 62 47 48 62 CA 59 EB 86
+0x0070 | 3F 3A FA 86 B5 BF A8 6D 06 16 50 82 C4 CE 62 9E
+0x0080 | 4E E6 4C C7 30 3E DE A1 0B D8 83 0B B6 B8 28 BC
+0x0090 | A9 EB 77 43 FC 7A 17 94 85 21 CA 37 6B 30 95 B5
+0x00A0 | 46 77 30 60 B7 12 D6 8C C5 54 85 29 D8 69 A9 6F
+0x00B0 | 12 4E 71 DF E3 E2 B1 6B 6B BF 9F FB 2E 57 30 A9
+0x00C0 | 69 76 C4 46 A2 DF FA 91 D9 50 74 55 1D 49 04 5A
+0x00D0 | 1C D6 86 68 7C B6 61 48 6C 96 E6 12 4C 27 AD BA
+0x00E0 | C7 51 99 8E D0 F0 ED 8E F6 65 79 79 A6 12 A1 95
+0x00F0 | DB C8 AE E3 B6 35 E6 8D BC 48 A3 7F AF 4A 28 8A
+0x0100 | 53 E2 7E 68 08 9F 67 77 98 52 DB 50 84 D6 5E 25
+0x0110 | E1 4A 99 58 34 C7 11 D6 43 FF C4 FD 9A 44 16 D1
+0x0120 | B2 FB 02 DB A1 89 69 34 C2 32 55 98 F9 9B B2 31
+0x0130 | 3F 49 59 0C 06 8C DB A5 B2 9D 7E 12 2F D0 87 94
+0x0140 | 44 E4 0A 76 EF 99 2D 91 18 39 50 3B 29 3B F5 2C
+0x0150 | 97 73 48 91 83 B0 A6 F3 4B 70 2F 1C 8F 3B 78 23
+0x0160 | C6 AA 86 46 43 1D D7 2A 23 5E 2C D9 48 0A F5 F5
+0x0170 | 2C D1 FB 3F F0 4B 78 37 E9 45 DD 72 CF 80 35 C3
+0x0180 | 95 07 F3 D9 06 E5 4A 58 76 03 6C 81 20 62 45 65
+0x0190 | 44 73 BC FE C1 9F 31 E5 DB 89 5C 6B 79 D8 68 90
+0x01A0 | D7 26 A8 A1 88 86 81 DC 9A 4F 40 A5 23 C7 DE BE
+0x01B0 | 6F 76 AB 79 16 51 21 67 83 2E F3 D6 27 1A 42 C2
+0x01C0 | 94 D1 5D 6C DB 4A 7A E2 CB 0B B0 68 0B BE 19 59
+0x01D0 | 00 50 FC C0 BD 9D F5 F5 F8 A8 17 19 D6 B3 E9 74
+0x01E0 | BA 50 E5 2C 45 7B F9 93 EA 5A F9 A9 30 B1 6F 5B
+0x01F0 | 36 24 1E 8D 55 57 F4 CC 67 B2 65 6A A9 36 26 D0
+0x0200 | 06 B8 E2 E3 73 8B D1 C0 1C 52 15 CA B5 AC 60 3E
+0x0210 | 36 42 F1 2C BD 99 77 AB A8 A9 A4 8E 9C 8B 84 DE
+0x0220 | 73 F0 91 29 97 AE DB AF D6 F8 5E 9B 86 B3 B3 03
+0x0230 | B3 AC 75 6F A6 11 69 2F 3D 3A CE FA 53 86 60 95
+0x0240 | 6C BB C5 4E F3
+
+An advantage to this format is the ability to configure the decoder at the start and jump into the middle of a stream. This is a common use case where the media is available on a random access medium such as a hard drive, and is therefore used in common container formats such as MP4 and MKV.
+*/
+
+var StartCodeBytes = []byte{0, 0, 1}
+var AUDBytes = []byte{0, 0, 0, 1, 0x9, 0xf0, 0, 0, 0, 1} // AUD
+
+func CheckNALUsType(b []byte) (typ int) {
+	_, typ = SplitNALUs(b)
+	return
+}
+
+const (
+	NALU_RAW = iota
+	NALU_AVCC
+	NALU_ANNEXB
+)
+
+func SplitNALUs(b []byte) (nalus [][]byte, typ int) {
+	if len(b) < 4 {
+		return [][]byte{b}, NALU_RAW
+	}
+
+	val3 := pio.U24BE(b)
+	val4 := pio.U32BE(b)
+
+	// maybe AVCC
+	if val4 <= uint32(len(b)) {
+		_val4 := val4
+		_b := b[4:]
+		nalus := [][]byte{}
+		for {
+			nalus = append(nalus, _b[:_val4])
+			_b = _b[_val4:]
+			if len(_b) < 4 {
+				break
+			}
+			_val4 = pio.U32BE(_b)
+			_b = _b[4:]
+			if _val4 > uint32(len(_b)) {
+				break
+			}
+		}
+		if len(_b) == 0 {
+			return nalus, NALU_AVCC
+		}
+	}
+
+	// is Annex B
+	if val3 == 1 || val4 == 1 {
+		_val3 := val3
+		_val4 := val4
+		start := 0
+		pos := 0
+		for {
+			if start != pos {
+				nalus = append(nalus, b[start:pos])
+			}
+			if _val3 == 1 {
+				pos += 3
+			} else if _val4 == 1 {
+				pos += 4
+			}
+			start = pos
+			if start == len(b) {
+				break
+			}
+			_val3 = 0
+			_val4 = 0
+			for pos < len(b) {
+				if pos+2 < len(b) && b[pos] == 0 {
+					_val3 = pio.U24BE(b[pos:])
+					if _val3 == 0 {
+						if pos+3 < len(b) {
+							_val4 = uint32(b[pos+3])
+							if _val4 == 1 {
+								break
+							}
+						}
+					} else if _val3 == 1 {
+						break
+					}
+					pos++
+				} else {
+					pos++
+				}
+			}
+		}
+		typ = NALU_ANNEXB
+		return
+	}
+
+	return [][]byte{b}, NALU_RAW
+}
+
+type SPSInfo struct {
+	ProfileIdc uint
+	LevelIdc   uint
+
+	MbWidth  uint
+	MbHeight uint
+
+	CropLeft   uint
+	CropRight  uint
+	CropTop    uint
+	CropBottom uint
+
+	Width  uint
+	Height uint
+}
+
+func ParseSPS(data []byte) (self SPSInfo, err error) {
+	r := &bits.GolombBitReader{R: bytes.NewReader(data)}
+
+	if _, err = r.ReadBits(8); err != nil {
+		return
+	}
+
+	if self.ProfileIdc, err = r.ReadBits(8); err != nil {
+		return
+	}
+
+	// constraint_set0_flag-constraint_set6_flag,reserved_zero_2bits
+	if _, err = r.ReadBits(8); err != nil {
+		return
+	}
+
+	// level_idc
+	if self.LevelIdc, err = r.ReadBits(8); err != nil {
+		return
+	}
+
+	// seq_parameter_set_id
+	if _, err = r.ReadExponentialGolombCode(); err != nil {
+		return
+	}
+
+	if self.ProfileIdc == 100 || self.ProfileIdc == 110 ||
+		self.ProfileIdc == 122 || self.ProfileIdc == 244 ||
+		self.ProfileIdc == 44 || self.ProfileIdc == 83 ||
+		self.ProfileIdc == 86 || self.ProfileIdc == 118 {
+
+		var chroma_format_idc uint
+		if chroma_format_idc, err = r.ReadExponentialGolombCode(); err != nil {
+			return
+		}
+
+		if chroma_format_idc == 3 {
+			// residual_colour_transform_flag
+			if _, err = r.ReadBit(); err != nil {
+				return
+			}
+		}
+
+		// bit_depth_luma_minus8
+		if _, err = r.ReadExponentialGolombCode(); err != nil {
+			return
+		}
+		// bit_depth_chroma_minus8
+		if _, err = r.ReadExponentialGolombCode(); err != nil {
+			return
+		}
+		// qpprime_y_zero_transform_bypass_flag
+		if _, err = r.ReadBit(); err != nil {
+			return
+		}
+
+		var seq_scaling_matrix_present_flag uint
+		if seq_scaling_matrix_present_flag, err = r.ReadBit(); err != nil {
+			return
+		}
+
+		if seq_scaling_matrix_present_flag != 0 {
+			for i := 0; i < 8; i++ {
+				var seq_scaling_list_present_flag uint
+				if seq_scaling_list_present_flag, err = r.ReadBit(); err != nil {
+					return
+				}
+				if seq_scaling_list_present_flag != 0 {
+					var sizeOfScalingList uint
+					if i < 6 {
+						sizeOfScalingList = 16
+					} else {
+						sizeOfScalingList = 64
+					}
+					lastScale := uint(8)
+					nextScale := uint(8)
+					for j := uint(0); j < sizeOfScalingList; j++ {
+						if nextScale != 0 {
+							var delta_scale uint
+							if delta_scale, err = r.ReadSE(); err != nil {
+								return
+							}
+							nextScale = (lastScale + delta_scale + 256) % 256
+						}
+						if nextScale != 0 {
+							lastScale = nextScale
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// log2_max_frame_num_minus4
+	if _, err = r.ReadExponentialGolombCode(); err != nil {
+		return
+	}
+
+	var pic_order_cnt_type uint
+	if pic_order_cnt_type, err = r.ReadExponentialGolombCode(); err != nil {
+		return
+	}
+	if pic_order_cnt_type == 0 {
+		// log2_max_pic_order_cnt_lsb_minus4
+		if _, err = r.ReadExponentialGolombCode(); err != nil {
+			return
+		}
+	} else if pic_order_cnt_type == 1 {
+		// delta_pic_order_always_zero_flag
+		if _, err = r.ReadBit(); err != nil {
+			return
+		}
+		// offset_for_non_ref_pic
+		if _, err = r.ReadSE(); err != nil {
+			return
+		}
+		// offset_for_top_to_bottom_field
+		if _, err = r.ReadSE(); err != nil {
+			return
+		}
+		var num_ref_frames_in_pic_order_cnt_cycle uint
+		if num_ref_frames_in_pic_order_cnt_cycle, err = r.ReadExponentialGolombCode(); err != nil {
+			return
+		}
+		for i := uint(0); i < num_ref_frames_in_pic_order_cnt_cycle; i++ {
+			if _, err = r.ReadSE(); err != nil {
+				return
+			}
+		}
+	}
+
+	// max_num_ref_frames
+	if _, err = r.ReadExponentialGolombCode(); err != nil {
+		return
+	}
+
+	// gaps_in_frame_num_value_allowed_flag
+	if _, err = r.ReadBit(); err != nil {
+		return
+	}
+
+	if self.MbWidth, err = r.ReadExponentialGolombCode(); err != nil {
+		return
+	}
+	self.MbWidth++
+
+	if self.MbHeight, err = r.ReadExponentialGolombCode(); err != nil {
+		return
+	}
+	self.MbHeight++
+
+	var frame_mbs_only_flag uint
+	if frame_mbs_only_flag, err = r.ReadBit(); err != nil {
+		return
+	}
+	if frame_mbs_only_flag == 0 {
+		// mb_adaptive_frame_field_flag
+		if _, err = r.ReadBit(); err != nil {
+			return
+		}
+	}
+
+	// direct_8x8_inference_flag
+	if _, err = r.ReadBit(); err != nil {
+		return
+	}
+
+	var frame_cropping_flag uint
+	if frame_cropping_flag, err = r.ReadBit(); err != nil {
+		return
+	}
+	if frame_cropping_flag != 0 {
+		if self.CropLeft, err = r.ReadExponentialGolombCode(); err != nil {
+			return
+		}
+		if self.CropRight, err = r.ReadExponentialGolombCode(); err != nil {
+			return
+		}
+		if self.CropTop, err = r.ReadExponentialGolombCode(); err != nil {
+			return
+		}
+		if self.CropBottom, err = r.ReadExponentialGolombCode(); err != nil {
+			return
+		}
+	}
+
+	self.Width = (self.MbWidth * 16) - self.CropLeft*2 - self.CropRight*2
+	self.Height = ((2 - frame_mbs_only_flag) * self.MbHeight * 16) - self.CropTop*2 - self.CropBottom*2
+
+	return
+}
+
+type CodecData struct {
+	Record     []byte
+	RecordInfo AVCDecoderConfRecord
+	SPSInfo    SPSInfo
+}
+
+func (self CodecData) Type() av.CodecType {
+	return av.H264
+}
+
+func (self CodecData) AVCDecoderConfRecordBytes() []byte {
+	return self.Record
+}
+
+func (self CodecData) SPS() []byte {
+	return self.RecordInfo.SPS[0]
+}
+
+func (self CodecData) PPS() []byte {
+	return self.RecordInfo.PPS[0]
+}
+
+func (self CodecData) Width() int {
+	return int(self.SPSInfo.Width)
+}
+
+func (self CodecData) Height() int {
+	return int(self.SPSInfo.Height)
+}
+
+func NewCodecDataFromAVCDecoderConfRecord(record []byte) (self CodecData, err error) {
+	self.Record = record
+	if _, err = (&self.RecordInfo).Unmarshal(record); err != nil {
+		return
+	}
+	if len(self.RecordInfo.SPS) == 0 {
+		err = fmt.Errorf("h264parser: no SPS found in AVCDecoderConfRecord")
+		return
+	}
+	if len(self.RecordInfo.PPS) == 0 {
+		err = fmt.Errorf("h264parser: no PPS found in AVCDecoderConfRecord")
+		return
+	}
+	if self.SPSInfo, err = ParseSPS(self.RecordInfo.SPS[0]); err != nil {
+		err = fmt.Errorf("h264parser: parse SPS failed(%s)", err)
+		return
+	}
+	return
+}
+
+func NewCodecDataFromSPSAndPPS(sps, pps []byte) (self CodecData, err error) {
+	recordinfo := AVCDecoderConfRecord{}
+	recordinfo.AVCProfileIndication = sps[1]
+	recordinfo.ProfileCompatibility = sps[2]
+	recordinfo.AVCLevelIndication = sps[3]
+	recordinfo.SPS = [][]byte{sps}
+	recordinfo.PPS = [][]byte{pps}
+	recordinfo.LengthSizeMinusOne = 3
+
+	buf := make([]byte, recordinfo.Len())
+	recordinfo.Marshal(buf)
+
+	self.RecordInfo = recordinfo
+	self.Record = buf
+
+	if self.SPSInfo, err = ParseSPS(sps); err != nil {
+		return
+	}
+	return
+}
+
+type AVCDecoderConfRecord struct {
+	AVCProfileIndication uint8
+	ProfileCompatibility uint8
+	AVCLevelIndication   uint8
+	LengthSizeMinusOne   uint8
+	SPS                  [][]byte
+	PPS                  [][]byte
+}
+
+var ErrDecconfInvalid = fmt.Errorf("h264parser: AVCDecoderConfRecord invalid")
+
+func (self *AVCDecoderConfRecord) Unmarshal(b []byte) (n int, err error) {
+	if len(b) < 7 {
+		err = ErrDecconfInvalid
+		return
+	}
+
+	self.AVCProfileIndication = b[1]
+	self.ProfileCompatibility = b[2]
+	self.AVCLevelIndication = b[3]
+	self.LengthSizeMinusOne = b[4] & 0x03
+	spscount := int(b[5] & 0x1f)
+	n += 6
+
+	for i := 0; i < spscount; i++ {
+		if len(b) < n+2 {
+			err = ErrDecconfInvalid
+			return
+		}
+		spslen := int(pio.U16BE(b[n:]))
+		n += 2
+
+		if len(b) < n+spslen {
+			err = ErrDecconfInvalid
+			return
+		}
+		self.SPS = append(self.SPS, b[n:n+spslen])
+		n += spslen
+	}
+
+	if len(b) < n+1 {
+		err = ErrDecconfInvalid
+		return
+	}
+	ppscount := int(b[n])
+	n++
+
+	for i := 0; i < ppscount; i++ {
+		if len(b) < n+2 {
+			err = ErrDecconfInvalid
+			return
+		}
+		ppslen := int(pio.U16BE(b[n:]))
+		n += 2
+
+		if len(b) < n+ppslen {
+			err = ErrDecconfInvalid
+			return
+		}
+		self.PPS = append(self.PPS, b[n:n+ppslen])
+		n += ppslen
+	}
+
+	return
+}
+
+func (self AVCDecoderConfRecord) Len() (n int) {
+	n = 7
+	for _, sps := range self.SPS {
+		n += 2 + len(sps)
+	}
+	for _, pps := range self.PPS {
+		n += 2 + len(pps)
+	}
+	return
+}
+
+func (self AVCDecoderConfRecord) Marshal(b []byte) (n int) {
+	b[0] = 1
+	b[1] = self.AVCProfileIndication
+	b[2] = self.ProfileCompatibility
+	b[3] = self.AVCLevelIndication
+	b[4] = self.LengthSizeMinusOne | 0xfc
+	b[5] = uint8(len(self.SPS)) | 0xe0
+	n += 6
+
+	for _, sps := range self.SPS {
+		pio.PutU16BE(b[n:], uint16(len(sps)))
+		n += 2
+		copy(b[n:], sps)
+		n += len(sps)
+	}
+
+	b[n] = uint8(len(self.PPS))
+	n++
+
+	for _, pps := range self.PPS {
+		pio.PutU16BE(b[n:], uint16(len(pps)))
+		n += 2
+		copy(b[n:], pps)
+		n += len(pps)
+	}
+
+	return
+}
+
+type SliceType uint
+
+func (self SliceType) String() string {
+	switch self {
+	case SLICE_P:
+		return "P"
+	case SLICE_B:
+		return "B"
+	case SLICE_I:
+		return "I"
+	}
+	return ""
+}
+
+const (
+	SLICE_P = iota + 1
+	SLICE_B
+	SLICE_I
+)
+
+func ParseSliceHeaderFromNALU(packet []byte) (sliceType SliceType, err error) {
+
+	if len(packet) <= 1 {
+		err = fmt.Errorf("h264parser: packet too short to parse slice header")
+		return
+	}
+
+	nal_unit_type := packet[0] & 0x1f
+	switch nal_unit_type {
+	case 1, 2, 5, 19:
+		// slice_layer_without_partitioning_rbsp
+		// slice_data_partition_a_layer_rbsp
+
+	default:
+		err = fmt.Errorf("h264parser: nal_unit_type=%d has no slice header", nal_unit_type)
+		return
+	}
+
+	r := &bits.GolombBitReader{R: bytes.NewReader(packet[1:])}
+
+	// first_mb_in_slice
+	if _, err = r.ReadExponentialGolombCode(); err != nil {
+		return
+	}
+
+	// slice_type
+	var u uint
+	if u, err = r.ReadExponentialGolombCode(); err != nil {
+		return
+	}
+
+	switch u {
+	case 0, 3, 5, 8:
+		sliceType = SLICE_P
+	case 1, 6:
+		sliceType = SLICE_B
+	case 2, 4, 7, 9:
+		sliceType = SLICE_I
+	default:
+		err = fmt.Errorf("h264parser: slice_type=%d invalid", u)
+		return
+	}
+
+	return
+}

codec/h264parser/parser_test.go

@@ -0,0 +1,23 @@
+
+package h264parser
+
+import (
+	"testing"
+	"encoding/hex"
+)
+
+func TestParser(t *testing.T) {
+	var ok bool
+	var nalus [][]byte
+
+	annexbFrame, _ := hex.DecodeString("00000001223322330000000122332233223300000133000001000001")
+	nalus, ok = SplitNALUs(annexbFrame)
+	t.Log(ok, len(nalus))
+
+	avccFrame, _ := hex.DecodeString(
+		"00000008aabbccaabbccaabb00000001aa",
+	)
+	nalus, ok = SplitNALUs(avccFrame)
+	t.Log(ok, len(nalus))
+}
+

format/aac/aac.go

@@ -0,0 +1,124 @@
+package aac
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/av/avutil"
+	"github.com/deepch/vdk/codec/aacparser"
+)
+
+type Muxer struct {
+	w       io.Writer
+	config  aacparser.MPEG4AudioConfig
+	adtshdr []byte
+}
+
+func NewMuxer(w io.Writer) *Muxer {
+	return &Muxer{
+		adtshdr: make([]byte, aacparser.ADTSHeaderLength),
+		w:       w,
+	}
+}
+
+func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) {
+	if len(streams) > 1 || streams[0].Type() != av.AAC {
+		err = fmt.Errorf("aac: must be only one aac stream")
+		return
+	}
+	self.config = streams[0].(aacparser.CodecData).Config
+	if self.config.ObjectType > aacparser.AOT_AAC_LTP {
+		err = fmt.Errorf("aac: AOT %d is not allowed in ADTS", self.config.ObjectType)
+	}
+	return
+}
+
+func (self *Muxer) WritePacket(pkt av.Packet) (err error) {
+	aacparser.FillADTSHeader(self.adtshdr, self.config, 1024, len(pkt.Data))
+	if _, err = self.w.Write(self.adtshdr); err != nil {
+		return
+	}
+	if _, err = self.w.Write(pkt.Data); err != nil {
+		return
+	}
+	return
+}
+
+func (self *Muxer) WriteTrailer() (err error) {
+	return
+}
+
+type Demuxer struct {
+	r         *bufio.Reader
+	config    aacparser.MPEG4AudioConfig
+	codecdata av.CodecData
+	ts        time.Duration
+}
+
+func NewDemuxer(r io.Reader) *Demuxer {
+	return &Demuxer{
+		r: bufio.NewReader(r),
+	}
+}
+
+func (self *Demuxer) Streams() (streams []av.CodecData, err error) {
+	if self.codecdata == nil {
+		var adtshdr []byte
+		var config aacparser.MPEG4AudioConfig
+		if adtshdr, err = self.r.Peek(9); err != nil {
+			return
+		}
+		if config, _, _, _, err = aacparser.ParseADTSHeader(adtshdr); err != nil {
+			return
+		}
+		if self.codecdata, err = aacparser.NewCodecDataFromMPEG4AudioConfig(config); err != nil {
+			return
+		}
+	}
+	streams = []av.CodecData{self.codecdata}
+	return
+}
+
+func (self *Demuxer) ReadPacket() (pkt av.Packet, err error) {
+	var adtshdr []byte
+	var config aacparser.MPEG4AudioConfig
+	var hdrlen, framelen, samples int
+	if adtshdr, err = self.r.Peek(9); err != nil {
+		return
+	}
+	if config, hdrlen, framelen, samples, err = aacparser.ParseADTSHeader(adtshdr); err != nil {
+		return
+	}
+
+	pkt.Data = make([]byte, framelen)
+	if _, err = io.ReadFull(self.r, pkt.Data); err != nil {
+		return
+	}
+	pkt.Data = pkt.Data[hdrlen:]
+
+	pkt.Time = self.ts
+	self.ts += time.Duration(samples) * time.Second / time.Duration(config.SampleRate)
+	return
+}
+
+func Handler(h *avutil.RegisterHandler) {
+	h.Ext = ".aac"
+
+	h.ReaderDemuxer = func(r io.Reader) av.Demuxer {
+		return NewDemuxer(r)
+	}
+
+	h.WriterMuxer = func(w io.Writer) av.Muxer {
+		return NewMuxer(w)
+	}
+
+	h.Probe = func(b []byte) bool {
+		_, _, _, _, err := aacparser.ParseADTSHeader(b)
+		return err == nil
+	}
+
+	h.CodecTypes = []av.CodecType{av.AAC}
+}

format/flv/flv.go

@@ -0,0 +1,495 @@
+package flv
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/av/avutil"
+	"github.com/deepch/vdk/codec"
+	"github.com/deepch/vdk/codec/aacparser"
+	"github.com/deepch/vdk/codec/fake"
+	"github.com/deepch/vdk/codec/h264parser"
+	"github.com/deepch/vdk/format/flv/flvio"
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+var MaxProbePacketCount = 20
+
+func NewMetadataByStreams(streams []av.CodecData) (metadata flvio.AMFMap, err error) {
+	metadata = flvio.AMFMap{}
+
+	for _, _stream := range streams {
+		typ := _stream.Type()
+		switch {
+		case typ.IsVideo():
+			stream := _stream.(av.VideoCodecData)
+			switch typ {
+			case av.H264:
+				metadata["videocodecid"] = flvio.VIDEO_H264
+
+			default:
+				err = fmt.Errorf("flv: metadata: unsupported video codecType=%v", stream.Type())
+				return
+			}
+
+			metadata["width"] = stream.Width()
+			metadata["height"] = stream.Height()
+			metadata["displayWidth"] = stream.Width()
+			metadata["displayHeight"] = stream.Height()
+
+		case typ.IsAudio():
+			stream := _stream.(av.AudioCodecData)
+			switch typ {
+			case av.AAC:
+				metadata["audiocodecid"] = flvio.SOUND_AAC
+
+			case av.SPEEX:
+				metadata["audiocodecid"] = flvio.SOUND_SPEEX
+
+			default:
+				err = fmt.Errorf("flv: metadata: unsupported audio codecType=%v", stream.Type())
+				return
+			}
+
+			metadata["audiosamplerate"] = stream.SampleRate()
+		}
+	}
+
+	return
+}
+
+type Prober struct {
+	HasAudio, HasVideo             bool
+	GotAudio, GotVideo             bool
+	VideoStreamIdx, AudioStreamIdx int
+	PushedCount                    int
+	Streams                        []av.CodecData
+	CachedPkts                     []av.Packet
+}
+
+func (self *Prober) CacheTag(_tag flvio.Tag, timestamp int32) {
+	pkt, _ := self.TagToPacket(_tag, timestamp)
+	self.CachedPkts = append(self.CachedPkts, pkt)
+}
+
+func (self *Prober) PushTag(tag flvio.Tag, timestamp int32) (err error) {
+	self.PushedCount++
+
+	if self.PushedCount > MaxProbePacketCount {
+		err = fmt.Errorf("flv: max probe packet count reached")
+		return
+	}
+
+	switch tag.Type {
+	case flvio.TAG_VIDEO:
+		switch tag.AVCPacketType {
+		case flvio.AVC_SEQHDR:
+			if !self.GotVideo {
+				var stream h264parser.CodecData
+				if stream, err = h264parser.NewCodecDataFromAVCDecoderConfRecord(tag.Data); err != nil {
+					err = fmt.Errorf("flv: h264 seqhdr invalid")
+					return
+				}
+				self.VideoStreamIdx = len(self.Streams)
+				self.Streams = append(self.Streams, stream)
+				self.GotVideo = true
+			}
+
+		case flvio.AVC_NALU:
+			self.CacheTag(tag, timestamp)
+		}
+
+	case flvio.TAG_AUDIO:
+		switch tag.SoundFormat {
+		case flvio.SOUND_AAC:
+			switch tag.AACPacketType {
+			case flvio.AAC_SEQHDR:
+				if !self.GotAudio {
+					var stream aacparser.CodecData
+					if stream, err = aacparser.NewCodecDataFromMPEG4AudioConfigBytes(tag.Data); err != nil {
+						err = fmt.Errorf("flv: aac seqhdr invalid")
+						return
+					}
+					self.AudioStreamIdx = len(self.Streams)
+					self.Streams = append(self.Streams, stream)
+					self.GotAudio = true
+				}
+
+			case flvio.AAC_RAW:
+				self.CacheTag(tag, timestamp)
+			}
+
+		case flvio.SOUND_SPEEX:
+			if !self.GotAudio {
+				stream := codec.NewSpeexCodecData(16000, tag.ChannelLayout())
+				self.AudioStreamIdx = len(self.Streams)
+				self.Streams = append(self.Streams, stream)
+				self.GotAudio = true
+				self.CacheTag(tag, timestamp)
+			}
+
+		case flvio.SOUND_NELLYMOSER:
+			if !self.GotAudio {
+				stream := fake.CodecData{
+					CodecType_:     av.NELLYMOSER,
+					SampleRate_:    16000,
+					SampleFormat_:  av.S16,
+					ChannelLayout_: tag.ChannelLayout(),
+				}
+				self.AudioStreamIdx = len(self.Streams)
+				self.Streams = append(self.Streams, stream)
+				self.GotAudio = true
+				self.CacheTag(tag, timestamp)
+			}
+
+		}
+	}
+
+	return
+}
+
+func (self *Prober) Probed() (ok bool) {
+	if self.HasAudio || self.HasVideo {
+		if self.HasAudio == self.GotAudio && self.HasVideo == self.GotVideo {
+			return true
+		}
+	} else {
+		if self.PushedCount == MaxProbePacketCount {
+			return true
+		}
+	}
+	return
+}
+
+func (self *Prober) TagToPacket(tag flvio.Tag, timestamp int32) (pkt av.Packet, ok bool) {
+	switch tag.Type {
+	case flvio.TAG_VIDEO:
+		pkt.Idx = int8(self.VideoStreamIdx)
+		switch tag.AVCPacketType {
+		case flvio.AVC_NALU:
+			ok = true
+			pkt.Data = tag.Data
+			pkt.CompositionTime = flvio.TsToTime(tag.CompositionTime)
+			pkt.IsKeyFrame = tag.FrameType == flvio.FRAME_KEY
+		}
+
+	case flvio.TAG_AUDIO:
+		pkt.Idx = int8(self.AudioStreamIdx)
+		switch tag.SoundFormat {
+		case flvio.SOUND_AAC:
+			switch tag.AACPacketType {
+			case flvio.AAC_RAW:
+				ok = true
+				pkt.Data = tag.Data
+			}
+
+		case flvio.SOUND_SPEEX:
+			ok = true
+			pkt.Data = tag.Data
+
+		case flvio.SOUND_NELLYMOSER:
+			ok = true
+			pkt.Data = tag.Data
+		}
+	}
+
+	pkt.Time = flvio.TsToTime(timestamp)
+	return
+}
+
+func (self *Prober) Empty() bool {
+	return len(self.CachedPkts) == 0
+}
+
+func (self *Prober) PopPacket() av.Packet {
+	pkt := self.CachedPkts[0]
+	self.CachedPkts = self.CachedPkts[1:]
+	return pkt
+}
+
+func CodecDataToTag(stream av.CodecData) (_tag flvio.Tag, ok bool, err error) {
+	switch stream.Type() {
+	case av.H264:
+		h264 := stream.(h264parser.CodecData)
+		tag := flvio.Tag{
+			Type:          flvio.TAG_VIDEO,
+			AVCPacketType: flvio.AVC_SEQHDR,
+			CodecID:       flvio.VIDEO_H264,
+			Data:          h264.AVCDecoderConfRecordBytes(),
+			FrameType:     flvio.FRAME_KEY,
+		}
+		ok = true
+		_tag = tag
+
+	case av.NELLYMOSER:
+	case av.SPEEX:
+
+	case av.AAC:
+		aac := stream.(aacparser.CodecData)
+		tag := flvio.Tag{
+			Type:          flvio.TAG_AUDIO,
+			SoundFormat:   flvio.SOUND_AAC,
+			SoundRate:     flvio.SOUND_44Khz,
+			AACPacketType: flvio.AAC_SEQHDR,
+			Data:          aac.MPEG4AudioConfigBytes(),
+		}
+		switch aac.SampleFormat().BytesPerSample() {
+		case 1:
+			tag.SoundSize = flvio.SOUND_8BIT
+		default:
+			tag.SoundSize = flvio.SOUND_16BIT
+		}
+		switch aac.ChannelLayout().Count() {
+		case 1:
+			tag.SoundType = flvio.SOUND_MONO
+		case 2:
+			tag.SoundType = flvio.SOUND_STEREO
+		}
+		ok = true
+		_tag = tag
+
+	default:
+		err = fmt.Errorf("flv: unspported codecType=%v", stream.Type())
+		return
+	}
+	return
+}
+
+func PacketToTag(pkt av.Packet, stream av.CodecData) (tag flvio.Tag, timestamp int32) {
+	switch stream.Type() {
+	case av.H264:
+		tag = flvio.Tag{
+			Type:            flvio.TAG_VIDEO,
+			AVCPacketType:   flvio.AVC_NALU,
+			CodecID:         flvio.VIDEO_H264,
+			Data:            pkt.Data,
+			CompositionTime: flvio.TimeToTs(pkt.CompositionTime),
+		}
+		if pkt.IsKeyFrame {
+			tag.FrameType = flvio.FRAME_KEY
+		} else {
+			tag.FrameType = flvio.FRAME_INTER
+		}
+
+	case av.AAC:
+		tag = flvio.Tag{
+			Type:          flvio.TAG_AUDIO,
+			SoundFormat:   flvio.SOUND_AAC,
+			SoundRate:     flvio.SOUND_44Khz,
+			AACPacketType: flvio.AAC_RAW,
+			Data:          pkt.Data,
+		}
+		astream := stream.(av.AudioCodecData)
+		switch astream.SampleFormat().BytesPerSample() {
+		case 1:
+			tag.SoundSize = flvio.SOUND_8BIT
+		default:
+			tag.SoundSize = flvio.SOUND_16BIT
+		}
+		switch astream.ChannelLayout().Count() {
+		case 1:
+			tag.SoundType = flvio.SOUND_MONO
+		case 2:
+			tag.SoundType = flvio.SOUND_STEREO
+		}
+
+	case av.SPEEX:
+		tag = flvio.Tag{
+			Type:        flvio.TAG_AUDIO,
+			SoundFormat: flvio.SOUND_SPEEX,
+			Data:        pkt.Data,
+		}
+
+	case av.NELLYMOSER:
+		tag = flvio.Tag{
+			Type:        flvio.TAG_AUDIO,
+			SoundFormat: flvio.SOUND_NELLYMOSER,
+			Data:        pkt.Data,
+		}
+	}
+
+	timestamp = flvio.TimeToTs(pkt.Time)
+	return
+}
+
+type Muxer struct {
+	bufw    writeFlusher
+	b       []byte
+	streams []av.CodecData
+}
+
+type writeFlusher interface {
+	io.Writer
+	Flush() error
+}
+
+func NewMuxerWriteFlusher(w writeFlusher) *Muxer {
+	return &Muxer{
+		bufw: w,
+		b:    make([]byte, 256),
+	}
+}
+
+func NewMuxer(w io.Writer) *Muxer {
+	return NewMuxerWriteFlusher(bufio.NewWriterSize(w, pio.RecommendBufioSize))
+}
+
+var CodecTypes = []av.CodecType{av.H264, av.AAC, av.SPEEX}
+
+func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) {
+	var flags uint8
+	for _, stream := range streams {
+		if stream.Type().IsVideo() {
+			flags |= flvio.FILE_HAS_VIDEO
+		} else if stream.Type().IsAudio() {
+			flags |= flvio.FILE_HAS_AUDIO
+		}
+	}
+
+	n := flvio.FillFileHeader(self.b, flags)
+	if _, err = self.bufw.Write(self.b[:n]); err != nil {
+		return
+	}
+
+	for _, stream := range streams {
+		var tag flvio.Tag
+		var ok bool
+		if tag, ok, err = CodecDataToTag(stream); err != nil {
+			return
+		}
+		if ok {
+			if err = flvio.WriteTag(self.bufw, tag, 0, self.b); err != nil {
+				return
+			}
+		}
+	}
+
+	self.streams = streams
+	return
+}
+
+func (self *Muxer) WritePacket(pkt av.Packet) (err error) {
+	stream := self.streams[pkt.Idx]
+	tag, timestamp := PacketToTag(pkt, stream)
+
+	if err = flvio.WriteTag(self.bufw, tag, timestamp, self.b); err != nil {
+		return
+	}
+	return
+}
+
+func (self *Muxer) WriteTrailer() (err error) {
+	if err = self.bufw.Flush(); err != nil {
+		return
+	}
+	return
+}
+
+type Demuxer struct {
+	prober *Prober
+	bufr   *bufio.Reader
+	b      []byte
+	stage  int
+}
+
+func NewDemuxer(r io.Reader) *Demuxer {
+	return &Demuxer{
+		bufr:   bufio.NewReaderSize(r, pio.RecommendBufioSize),
+		prober: &Prober{},
+		b:      make([]byte, 256),
+	}
+}
+
+func (self *Demuxer) prepare() (err error) {
+	for self.stage < 2 {
+		switch self.stage {
+		case 0:
+			if _, err = io.ReadFull(self.bufr, self.b[:flvio.FileHeaderLength]); err != nil {
+				return
+			}
+			var flags uint8
+			var skip int
+			if flags, skip, err = flvio.ParseFileHeader(self.b); err != nil {
+				return
+			}
+			if _, err = self.bufr.Discard(skip); err != nil {
+				return
+			}
+			if flags&flvio.FILE_HAS_AUDIO != 0 {
+				self.prober.HasAudio = true
+			}
+			if flags&flvio.FILE_HAS_VIDEO != 0 {
+				self.prober.HasVideo = true
+			}
+			self.stage++
+
+		case 1:
+			for !self.prober.Probed() {
+				var tag flvio.Tag
+				var timestamp int32
+				if tag, timestamp, err = flvio.ReadTag(self.bufr, self.b); err != nil {
+					return
+				}
+				if err = self.prober.PushTag(tag, timestamp); err != nil {
+					return
+				}
+			}
+			self.stage++
+		}
+	}
+	return
+}
+
+func (self *Demuxer) Streams() (streams []av.CodecData, err error) {
+	if err = self.prepare(); err != nil {
+		return
+	}
+	streams = self.prober.Streams
+	return
+}
+
+func (self *Demuxer) ReadPacket() (pkt av.Packet, err error) {
+	if err = self.prepare(); err != nil {
+		return
+	}
+
+	if !self.prober.Empty() {
+		pkt = self.prober.PopPacket()
+		return
+	}
+
+	for {
+		var tag flvio.Tag
+		var timestamp int32
+		if tag, timestamp, err = flvio.ReadTag(self.bufr, self.b); err != nil {
+			return
+		}
+
+		var ok bool
+		if pkt, ok = self.prober.TagToPacket(tag, timestamp); ok {
+			return
+		}
+	}
+
+	return
+}
+
+func Handler(h *avutil.RegisterHandler) {
+	h.Probe = func(b []byte) bool {
+		return b[0] == 'F' && b[1] == 'L' && b[2] == 'V'
+	}
+
+	h.Ext = ".flv"
+
+	h.ReaderDemuxer = func(r io.Reader) av.Demuxer {
+		return NewDemuxer(r)
+	}
+
+	h.WriterMuxer = func(w io.Writer) av.Muxer {
+		return NewMuxer(w)
+	}
+
+	h.CodecTypes = CodecTypes
+}

format/flv/flvio/amf0.go

@@ -0,0 +1,467 @@
+package flvio
+
+import (
+	"fmt"
+	"math"
+	"strings"
+	"time"
+
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+type AMF0ParseError struct {
+	Offset  int
+	Message string
+	Next    *AMF0ParseError
+}
+
+func (self *AMF0ParseError) Error() string {
+	s := []string{}
+	for p := self; p != nil; p = p.Next {
+		s = append(s, fmt.Sprintf("%s:%d", p.Message, p.Offset))
+	}
+	return "amf0 parse error: " + strings.Join(s, ",")
+}
+
+func amf0ParseErr(message string, offset int, err error) error {
+	next, _ := err.(*AMF0ParseError)
+	return &AMF0ParseError{
+		Offset:  offset,
+		Message: message,
+		Next:    next,
+	}
+}
+
+type AMFMap map[string]interface{}
+type AMFArray []interface{}
+type AMFECMAArray map[string]interface{}
+
+func parseBEFloat64(b []byte) float64 {
+	return math.Float64frombits(pio.U64BE(b))
+}
+
+func fillBEFloat64(b []byte, f float64) int {
+	pio.PutU64BE(b, math.Float64bits(f))
+	return 8
+}
+
+const lenAMF0Number = 9
+
+func fillAMF0Number(b []byte, f float64) int {
+	b[0] = numbermarker
+	fillBEFloat64(b[1:], f)
+	return lenAMF0Number
+}
+
+const (
+	amf3undefinedmarker = iota
+	amf3nullmarker
+	amf3falsemarker
+	amf3truemarker
+	amf3integermarker
+	amf3doublemarker
+	amf3stringmarker
+	amf3xmldocmarker
+	amf3datemarker
+	amf3arraymarker
+	amf3objectmarker
+	amf3xmlmarker
+	amf3bytearraymarker
+	amf3vectorintmarker
+	amf3vectoruintmarker
+	amf3vectordoublemarker
+	amf3vectorobjectmarker
+	amf3dictionarymarker
+)
+
+const (
+	numbermarker = iota
+	booleanmarker
+	stringmarker
+	objectmarker
+	movieclipmarker
+	nullmarker
+	undefinedmarker
+	referencemarker
+	ecmaarraymarker
+	objectendmarker
+	strictarraymarker
+	datemarker
+	longstringmarker
+	unsupportedmarker
+	recordsetmarker
+	xmldocumentmarker
+	typedobjectmarker
+	avmplusobjectmarker
+)
+
+func LenAMF0Val(_val interface{}) (n int) {
+	switch val := _val.(type) {
+	case int8:
+		n += lenAMF0Number
+	case int16:
+		n += lenAMF0Number
+	case int32:
+		n += lenAMF0Number
+	case int64:
+		n += lenAMF0Number
+	case int:
+		n += lenAMF0Number
+	case uint8:
+		n += lenAMF0Number
+	case uint16:
+		n += lenAMF0Number
+	case uint32:
+		n += lenAMF0Number
+	case uint64:
+		n += lenAMF0Number
+	case uint:
+		n += lenAMF0Number
+	case float32:
+		n += lenAMF0Number
+	case float64:
+		n += lenAMF0Number
+
+	case string:
+		u := len(val)
+		if u <= 65536 {
+			n += 3
+		} else {
+			n += 5
+		}
+		n += int(u)
+
+	case AMFECMAArray:
+		n += 5
+		for k, v := range val {
+			n += 2 + len(k)
+			n += LenAMF0Val(v)
+		}
+		n += 3
+
+	case AMFMap:
+		n++
+		for k, v := range val {
+			if len(k) > 0 {
+				n += 2 + len(k)
+				n += LenAMF0Val(v)
+			}
+		}
+		n += 3
+
+	case AMFArray:
+		n += 5
+		for _, v := range val {
+			n += LenAMF0Val(v)
+		}
+
+	case time.Time:
+		n += 1 + 8 + 2
+
+	case bool:
+		n += 2
+
+	case nil:
+		n++
+	}
+
+	return
+}
+
+func FillAMF0Val(b []byte, _val interface{}) (n int) {
+	switch val := _val.(type) {
+	case int8:
+		n += fillAMF0Number(b[n:], float64(val))
+	case int16:
+		n += fillAMF0Number(b[n:], float64(val))
+	case int32:
+		n += fillAMF0Number(b[n:], float64(val))
+	case int64:
+		n += fillAMF0Number(b[n:], float64(val))
+	case int:
+		n += fillAMF0Number(b[n:], float64(val))
+	case uint8:
+		n += fillAMF0Number(b[n:], float64(val))
+	case uint16:
+		n += fillAMF0Number(b[n:], float64(val))
+	case uint32:
+		n += fillAMF0Number(b[n:], float64(val))
+	case uint64:
+		n += fillAMF0Number(b[n:], float64(val))
+	case uint:
+		n += fillAMF0Number(b[n:], float64(val))
+	case float32:
+		n += fillAMF0Number(b[n:], float64(val))
+	case float64:
+		n += fillAMF0Number(b[n:], float64(val))
+
+	case string:
+		u := len(val)
+		if u <= 65536 {
+			b[n] = stringmarker
+			n++
+			pio.PutU16BE(b[n:], uint16(u))
+			n += 2
+		} else {
+			b[n] = longstringmarker
+			n++
+			pio.PutU32BE(b[n:], uint32(u))
+			n += 4
+		}
+		copy(b[n:], []byte(val))
+		n += len(val)
+
+	case AMFECMAArray:
+		b[n] = ecmaarraymarker
+		n++
+		pio.PutU32BE(b[n:], uint32(len(val)))
+		n += 4
+		for k, v := range val {
+			pio.PutU16BE(b[n:], uint16(len(k)))
+			n += 2
+			copy(b[n:], []byte(k))
+			n += len(k)
+			n += FillAMF0Val(b[n:], v)
+		}
+		pio.PutU24BE(b[n:], 0x000009)
+		n += 3
+
+	case AMFMap:
+		b[n] = objectmarker
+		n++
+		for k, v := range val {
+			if len(k) > 0 {
+				pio.PutU16BE(b[n:], uint16(len(k)))
+				n += 2
+				copy(b[n:], []byte(k))
+				n += len(k)
+				n += FillAMF0Val(b[n:], v)
+			}
+		}
+		pio.PutU24BE(b[n:], 0x000009)
+		n += 3
+
+	case AMFArray:
+		b[n] = strictarraymarker
+		n++
+		pio.PutU32BE(b[n:], uint32(len(val)))
+		n += 4
+		for _, v := range val {
+			n += FillAMF0Val(b[n:], v)
+		}
+
+	case time.Time:
+		b[n] = datemarker
+		n++
+		u := val.UnixNano()
+		f := float64(u / 1000000)
+		n += fillBEFloat64(b[n:], f)
+		pio.PutU16BE(b[n:], uint16(0))
+		n += 2
+
+	case bool:
+		b[n] = booleanmarker
+		n++
+		var u uint8
+		if val {
+			u = 1
+		} else {
+			u = 0
+		}
+		b[n] = u
+		n++
+
+	case nil:
+		b[n] = nullmarker
+		n++
+	}
+
+	return
+}
+
+func ParseAMF0Val(b []byte) (val interface{}, n int, err error) {
+	return parseAMF0Val(b, 0)
+}
+
+func parseAMF0Val(b []byte, offset int) (val interface{}, n int, err error) {
+	if len(b) < n+1 {
+		err = amf0ParseErr("marker", offset+n, err)
+		return
+	}
+	marker := b[n]
+	n++
+
+	switch marker {
+	case numbermarker:
+		if len(b) < n+8 {
+			err = amf0ParseErr("number", offset+n, err)
+			return
+		}
+		val = parseBEFloat64(b[n:])
+		n += 8
+
+	case booleanmarker:
+		if len(b) < n+1 {
+			err = amf0ParseErr("boolean", offset+n, err)
+			return
+		}
+		val = b[n] != 0
+		n++
+
+	case stringmarker:
+		if len(b) < n+2 {
+			err = amf0ParseErr("string.length", offset+n, err)
+			return
+		}
+		length := int(pio.U16BE(b[n:]))
+		n += 2
+
+		if len(b) < n+length {
+			err = amf0ParseErr("string.body", offset+n, err)
+			return
+		}
+		val = string(b[n : n+length])
+		n += length
+
+	case objectmarker:
+		obj := AMFMap{}
+		for {
+			if len(b) < n+2 {
+				err = amf0ParseErr("object.key.length", offset+n, err)
+				return
+			}
+			length := int(pio.U16BE(b[n:]))
+			n += 2
+			if length == 0 {
+				break
+			}
+
+			if len(b) < n+length {
+				err = amf0ParseErr("object.key.body", offset+n, err)
+				return
+			}
+			okey := string(b[n : n+length])
+			n += length
+
+			var nval int
+			var oval interface{}
+			if oval, nval, err = parseAMF0Val(b[n:], offset+n); err != nil {
+				err = amf0ParseErr("object.val", offset+n, err)
+				return
+			}
+			n += nval
+
+			obj[okey] = oval
+		}
+		if len(b) < n+1 {
+			err = amf0ParseErr("object.end", offset+n, err)
+			return
+		}
+		n++
+		val = obj
+
+	case nullmarker:
+	case undefinedmarker:
+
+	case ecmaarraymarker:
+		if len(b) < n+4 {
+			err = amf0ParseErr("array.count", offset+n, err)
+			return
+		}
+		n += 4
+
+		obj := AMFMap{}
+		for {
+			if len(b) < n+2 {
+				err = amf0ParseErr("array.key.length", offset+n, err)
+				return
+			}
+			length := int(pio.U16BE(b[n:]))
+			n += 2
+
+			if length == 0 {
+				break
+			}
+
+			if len(b) < n+length {
+				err = amf0ParseErr("array.key.body", offset+n, err)
+				return
+			}
+			okey := string(b[n : n+length])
+			n += length
+
+			var nval int
+			var oval interface{}
+			if oval, nval, err = parseAMF0Val(b[n:], offset+n); err != nil {
+				err = amf0ParseErr("array.val", offset+n, err)
+				return
+			}
+			n += nval
+
+			obj[okey] = oval
+		}
+		if len(b) < n+1 {
+			err = amf0ParseErr("array.end", offset+n, err)
+			return
+		}
+		n += 1
+		val = obj
+
+	case objectendmarker:
+		if len(b) < n+3 {
+			err = amf0ParseErr("objectend", offset+n, err)
+			return
+		}
+		n += 3
+
+	case strictarraymarker:
+		if len(b) < n+4 {
+			err = amf0ParseErr("strictarray.count", offset+n, err)
+			return
+		}
+		count := int(pio.U32BE(b[n:]))
+		n += 4
+
+		obj := make(AMFArray, count)
+		for i := 0; i < int(count); i++ {
+			var nval int
+			if obj[i], nval, err = parseAMF0Val(b[n:], offset+n); err != nil {
+				err = amf0ParseErr("strictarray.val", offset+n, err)
+				return
+			}
+			n += nval
+		}
+		val = obj
+
+	case datemarker:
+		if len(b) < n+8+2 {
+			err = amf0ParseErr("date", offset+n, err)
+			return
+		}
+		ts := parseBEFloat64(b[n:])
+		n += 8 + 2
+
+		val = time.Unix(int64(ts/1000), (int64(ts)%1000)*1000000)
+
+	case longstringmarker:
+		if len(b) < n+4 {
+			err = amf0ParseErr("longstring.length", offset+n, err)
+			return
+		}
+		length := int(pio.U32BE(b[n:]))
+		n += 4
+
+		if len(b) < n+length {
+			err = amf0ParseErr("longstring.body", offset+n, err)
+			return
+		}
+		val = string(b[n : n+length])
+		n += length
+
+	default:
+		err = amf0ParseErr(fmt.Sprintf("invalidmarker=%d", marker), offset+n, err)
+		return
+	}
+
+	return
+}

format/flv/flvio/flvio.go

@@ -0,0 +1,411 @@
+package flvio
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+func TsToTime(ts int32) time.Duration {
+	return time.Millisecond * time.Duration(ts)
+}
+
+func TimeToTs(tm time.Duration) int32 {
+	return int32(tm / time.Millisecond)
+}
+
+const MaxTagSubHeaderLength = 16
+
+const (
+	TAG_AUDIO      = 8
+	TAG_VIDEO      = 9
+	TAG_SCRIPTDATA = 18
+)
+
+const (
+	SOUND_MP3                   = 2
+	SOUND_NELLYMOSER_16KHZ_MONO = 4
+	SOUND_NELLYMOSER_8KHZ_MONO  = 5
+	SOUND_NELLYMOSER            = 6
+	SOUND_ALAW                  = 7
+	SOUND_MULAW                 = 8
+	SOUND_AAC                   = 10
+	SOUND_SPEEX                 = 11
+
+	SOUND_5_5Khz = 0
+	SOUND_11Khz  = 1
+	SOUND_22Khz  = 2
+	SOUND_44Khz  = 3
+
+	SOUND_8BIT  = 0
+	SOUND_16BIT = 1
+
+	SOUND_MONO   = 0
+	SOUND_STEREO = 1
+
+	AAC_SEQHDR = 0
+	AAC_RAW    = 1
+)
+
+const (
+	AVC_SEQHDR = 0
+	AVC_NALU   = 1
+	AVC_EOS    = 2
+
+	FRAME_KEY   = 1
+	FRAME_INTER = 2
+
+	VIDEO_H264 = 7
+)
+
+type Tag struct {
+	Type uint8
+
+	/*
+		SoundFormat: UB[4]
+		0 = Linear PCM, platform endian
+		1 = ADPCM
+		2 = MP3
+		3 = Linear PCM, little endian
+		4 = Nellymoser 16-kHz mono
+		5 = Nellymoser 8-kHz mono
+		6 = Nellymoser
+		7 = G.711 A-law logarithmic PCM
+		8 = G.711 mu-law logarithmic PCM
+		9 = reserved
+		10 = AAC
+		11 = Speex
+		14 = MP3 8-Khz
+		15 = Device-specific sound
+		Formats 7, 8, 14, and 15 are reserved for internal use
+		AAC is supported in Flash Player 9,0,115,0 and higher.
+		Speex is supported in Flash Player 10 and higher.
+	*/
+	SoundFormat uint8
+
+	/*
+		SoundRate: UB[2]
+		Sampling rate
+		0 = 5.5-kHz For AAC: always 3
+		1 = 11-kHz
+		2 = 22-kHz
+		3 = 44-kHz
+	*/
+	SoundRate uint8
+
+	/*
+		SoundSize: UB[1]
+		0 = snd8Bit
+		1 = snd16Bit
+		Size of each sample.
+		This parameter only pertains to uncompressed formats.
+		Compressed formats always decode to 16 bits internally
+	*/
+	SoundSize uint8
+
+	/*
+		SoundType: UB[1]
+		0 = sndMono
+		1 = sndStereo
+		Mono or stereo sound For Nellymoser: always 0
+		For AAC: always 1
+	*/
+	SoundType uint8
+
+	/*
+		0: AAC sequence header
+		1: AAC raw
+	*/
+	AACPacketType uint8
+
+	/*
+		1: keyframe (for AVC, a seekable frame)
+		2: inter frame (for AVC, a non- seekable frame)
+		3: disposable inter frame (H.263 only)
+		4: generated keyframe (reserved for server use only)
+		5: video info/command frame
+	*/
+	FrameType uint8
+
+	/*
+		1: JPEG (currently unused)
+		2: Sorenson H.263
+		3: Screen video
+		4: On2 VP6
+		5: On2 VP6 with alpha channel
+		6: Screen video version 2
+		7: AVC
+	*/
+	CodecID uint8
+
+	/*
+		0: AVC sequence header
+		1: AVC NALU
+		2: AVC end of sequence (lower level NALU sequence ender is not required or supported)
+	*/
+	AVCPacketType uint8
+
+	CompositionTime int32
+
+	Data []byte
+}
+
+func (self Tag) ChannelLayout() av.ChannelLayout {
+	if self.SoundType == SOUND_MONO {
+		return av.CH_MONO
+	} else {
+		return av.CH_STEREO
+	}
+}
+
+func (self *Tag) audioParseHeader(b []byte) (n int, err error) {
+	if len(b) < n+1 {
+		err = fmt.Errorf("audiodata: parse invalid")
+		return
+	}
+
+	flags := b[n]
+	n++
+	self.SoundFormat = flags >> 4
+	self.SoundRate = (flags >> 2) & 0x3
+	self.SoundSize = (flags >> 1) & 0x1
+	self.SoundType = flags & 0x1
+
+	switch self.SoundFormat {
+	case SOUND_AAC:
+		if len(b) < n+1 {
+			err = fmt.Errorf("audiodata: parse invalid")
+			return
+		}
+		self.AACPacketType = b[n]
+		n++
+	}
+
+	return
+}
+
+func (self Tag) audioFillHeader(b []byte) (n int) {
+	var flags uint8
+	flags |= self.SoundFormat << 4
+	flags |= self.SoundRate << 2
+	flags |= self.SoundSize << 1
+	flags |= self.SoundType
+	b[n] = flags
+	n++
+
+	switch self.SoundFormat {
+	case SOUND_AAC:
+		b[n] = self.AACPacketType
+		n++
+	}
+
+	return
+}
+
+func (self *Tag) videoParseHeader(b []byte) (n int, err error) {
+	if len(b) < n+1 {
+		err = fmt.Errorf("videodata: parse invalid")
+		return
+	}
+	flags := b[n]
+	self.FrameType = flags >> 4
+	self.CodecID = flags & 0xf
+	n++
+
+	if self.FrameType == FRAME_INTER || self.FrameType == FRAME_KEY {
+		if len(b) < n+4 {
+			err = fmt.Errorf("videodata: parse invalid")
+			return
+		}
+		self.AVCPacketType = b[n]
+		n++
+
+		self.CompositionTime = pio.I24BE(b[n:])
+		n += 3
+	}
+
+	return
+}
+
+func (self Tag) videoFillHeader(b []byte) (n int) {
+	flags := self.FrameType<<4 | self.CodecID
+	b[n] = flags
+	n++
+	b[n] = self.AVCPacketType
+	n++
+	pio.PutI24BE(b[n:], self.CompositionTime)
+	n += 3
+	return
+}
+
+func (self Tag) FillHeader(b []byte) (n int) {
+	switch self.Type {
+	case TAG_AUDIO:
+		return self.audioFillHeader(b)
+
+	case TAG_VIDEO:
+		return self.videoFillHeader(b)
+	}
+
+	return
+}
+
+func (self *Tag) ParseHeader(b []byte) (n int, err error) {
+	switch self.Type {
+	case TAG_AUDIO:
+		return self.audioParseHeader(b)
+
+	case TAG_VIDEO:
+		return self.videoParseHeader(b)
+	}
+
+	return
+}
+
+const (
+	// TypeFlagsReserved UB[5]
+	// TypeFlagsAudio    UB[1] Audio tags are present
+	// TypeFlagsReserved UB[1] Must be 0
+	// TypeFlagsVideo    UB[1] Video tags are present
+	FILE_HAS_AUDIO = 0x4
+	FILE_HAS_VIDEO = 0x1
+)
+
+const TagHeaderLength = 11
+const TagTrailerLength = 4
+
+func ParseTagHeader(b []byte) (tag Tag, ts int32, datalen int, err error) {
+	tagtype := b[0]
+
+	switch tagtype {
+	case TAG_AUDIO, TAG_VIDEO, TAG_SCRIPTDATA:
+		tag = Tag{Type: tagtype}
+
+	default:
+		err = fmt.Errorf("flvio: ReadTag tagtype=%d invalid", tagtype)
+		return
+	}
+
+	datalen = int(pio.U24BE(b[1:4]))
+
+	var tslo uint32
+	var tshi uint8
+	tslo = pio.U24BE(b[4:7])
+	tshi = b[7]
+	ts = int32(tslo | uint32(tshi)<<24)
+
+	return
+}
+
+func ReadTag(r io.Reader, b []byte) (tag Tag, ts int32, err error) {
+	if _, err = io.ReadFull(r, b[:TagHeaderLength]); err != nil {
+		return
+	}
+	var datalen int
+	if tag, ts, datalen, err = ParseTagHeader(b); err != nil {
+		return
+	}
+
+	data := make([]byte, datalen)
+	if _, err = io.ReadFull(r, data); err != nil {
+		return
+	}
+
+	var n int
+	if n, err = (&tag).ParseHeader(data); err != nil {
+		return
+	}
+	tag.Data = data[n:]
+
+	if _, err = io.ReadFull(r, b[:4]); err != nil {
+		return
+	}
+	return
+}
+
+func FillTagHeader(b []byte, tagtype uint8, datalen int, ts int32) (n int) {
+	b[n] = tagtype
+	n++
+	pio.PutU24BE(b[n:], uint32(datalen))
+	n += 3
+	pio.PutU24BE(b[n:], uint32(ts&0xffffff))
+	n += 3
+	b[n] = uint8(ts >> 24)
+	n++
+	pio.PutI24BE(b[n:], 0)
+	n += 3
+	return
+}
+
+func FillTagTrailer(b []byte, datalen int) (n int) {
+	pio.PutU32BE(b[n:], uint32(datalen+TagHeaderLength))
+	n += 4
+	return
+}
+
+func WriteTag(w io.Writer, tag Tag, ts int32, b []byte) (err error) {
+	data := tag.Data
+
+	n := tag.FillHeader(b[TagHeaderLength:])
+	datalen := len(data) + n
+
+	n += FillTagHeader(b, tag.Type, datalen, ts)
+
+	if _, err = w.Write(b[:n]); err != nil {
+		return
+	}
+
+	if _, err = w.Write(data); err != nil {
+		return
+	}
+
+	n = FillTagTrailer(b, datalen)
+	if _, err = w.Write(b[:n]); err != nil {
+		return
+	}
+
+	return
+}
+
+const FileHeaderLength = 9
+
+func FillFileHeader(b []byte, flags uint8) (n int) {
+	// 'FLV', version 1
+	pio.PutU32BE(b[n:], 0x464c5601)
+	n += 4
+
+	b[n] = flags
+	n++
+
+	// DataOffset: UI32 Offset in bytes from start of file to start of body (that is, size of header)
+	// The DataOffset field usually has a value of 9 for FLV version 1.
+	pio.PutU32BE(b[n:], 9)
+	n += 4
+
+	// PreviousTagSize0: UI32 Always 0
+	pio.PutU32BE(b[n:], 0)
+	n += 4
+
+	return
+}
+
+func ParseFileHeader(b []byte) (flags uint8, skip int, err error) {
+	flv := pio.U24BE(b[0:3])
+	if flv != 0x464c56 { // 'FLV'
+		err = fmt.Errorf("flvio: file header cc3 invalid")
+		return
+	}
+
+	flags = b[4]
+
+	skip = int(pio.U32BE(b[5:9])) - 9 + 4
+	if skip < 0 {
+		err = fmt.Errorf("flvio: file header datasize invalid")
+		return
+	}
+
+	return
+}

format/format.go

@@ -0,0 +1,20 @@
+package format
+
+import (
+	"github.com/deepch/vdk/av/avutil"
+	"github.com/deepch/vdk/format/aac"
+	"github.com/deepch/vdk/format/flv"
+	"github.com/deepch/vdk/format/mp4"
+	"github.com/deepch/vdk/format/rtmp"
+	"github.com/deepch/vdk/format/rtsp"
+	"github.com/deepch/vdk/format/ts"
+)
+
+func RegisterAll() {
+	avutil.DefaultHandlers.Add(mp4.Handler)
+	avutil.DefaultHandlers.Add(ts.Handler)
+	avutil.DefaultHandlers.Add(rtmp.Handler)
+	avutil.DefaultHandlers.Add(rtsp.Handler)
+	avutil.DefaultHandlers.Add(flv.Handler)
+	avutil.DefaultHandlers.Add(aac.Handler)
+}

format/mp4/demuxer.go

@@ -0,0 +1,446 @@
+package mp4
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/codec/aacparser"
+	"github.com/deepch/vdk/codec/h264parser"
+	"github.com/deepch/vdk/format/mp4/mp4io"
+)
+
+type Demuxer struct {
+	r         io.ReadSeeker
+	streams   []*Stream
+	movieAtom *mp4io.Movie
+}
+
+func NewDemuxer(r io.ReadSeeker) *Demuxer {
+	return &Demuxer{
+		r: r,
+	}
+}
+
+func (self *Demuxer) Streams() (streams []av.CodecData, err error) {
+	if err = self.probe(); err != nil {
+		return
+	}
+	for _, stream := range self.streams {
+		streams = append(streams, stream.CodecData)
+	}
+	return
+}
+
+func (self *Demuxer) readat(pos int64, b []byte) (err error) {
+	if _, err = self.r.Seek(pos, 0); err != nil {
+		return
+	}
+	if _, err = io.ReadFull(self.r, b); err != nil {
+		return
+	}
+	return
+}
+
+func (self *Demuxer) probe() (err error) {
+	if self.movieAtom != nil {
+		return
+	}
+
+	var moov *mp4io.Movie
+	var atoms []mp4io.Atom
+
+	if atoms, err = mp4io.ReadFileAtoms(self.r); err != nil {
+		return
+	}
+	if _, err = self.r.Seek(0, 0); err != nil {
+		return
+	}
+
+	for _, atom := range atoms {
+		if atom.Tag() == mp4io.MOOV {
+			moov = atom.(*mp4io.Movie)
+		}
+	}
+
+	if moov == nil {
+		err = fmt.Errorf("mp4: 'moov' atom not found")
+		return
+	}
+
+	self.streams = []*Stream{}
+	for i, atrack := range moov.Tracks {
+		stream := &Stream{
+			trackAtom: atrack,
+			demuxer:   self,
+			idx:       i,
+		}
+		if atrack.Media != nil && atrack.Media.Info != nil && atrack.Media.Info.Sample != nil {
+			stream.sample = atrack.Media.Info.Sample
+			stream.timeScale = int64(atrack.Media.Header.TimeScale)
+		} else {
+			err = fmt.Errorf("mp4: sample table not found")
+			return
+		}
+
+		if avc1 := atrack.GetAVC1Conf(); avc1 != nil {
+			if stream.CodecData, err = h264parser.NewCodecDataFromAVCDecoderConfRecord(avc1.Data); err != nil {
+				return
+			}
+			self.streams = append(self.streams, stream)
+		} else if esds := atrack.GetElemStreamDesc(); esds != nil {
+			if stream.CodecData, err = aacparser.NewCodecDataFromMPEG4AudioConfigBytes(esds.DecConfig); err != nil {
+				return
+			}
+			self.streams = append(self.streams, stream)
+		}
+	}
+
+	self.movieAtom = moov
+	return
+}
+
+func (self *Stream) setSampleIndex(index int) (err error) {
+	found := false
+	start := 0
+	self.chunkGroupIndex = 0
+
+	for self.chunkIndex = range self.sample.ChunkOffset.Entries {
+		if self.chunkGroupIndex+1 < len(self.sample.SampleToChunk.Entries) &&
+			uint32(self.chunkIndex+1) == self.sample.SampleToChunk.Entries[self.chunkGroupIndex+1].FirstChunk {
+			self.chunkGroupIndex++
+		}
+		n := int(self.sample.SampleToChunk.Entries[self.chunkGroupIndex].SamplesPerChunk)
+		if index >= start && index < start+n {
+			found = true
+			self.sampleIndexInChunk = index - start
+			break
+		}
+		start += n
+	}
+	if !found {
+		err = fmt.Errorf("mp4: stream[%d]: cannot locate sample index in chunk", self.idx)
+		return
+	}
+
+	if self.sample.SampleSize.SampleSize != 0 {
+		self.sampleOffsetInChunk = int64(self.sampleIndexInChunk) * int64(self.sample.SampleSize.SampleSize)
+	} else {
+		if index >= len(self.sample.SampleSize.Entries) {
+			err = fmt.Errorf("mp4: stream[%d]: sample index out of range", self.idx)
+			return
+		}
+		self.sampleOffsetInChunk = int64(0)
+		for i := index - self.sampleIndexInChunk; i < index; i++ {
+			self.sampleOffsetInChunk += int64(self.sample.SampleSize.Entries[i])
+		}
+	}
+
+	self.dts = int64(0)
+	start = 0
+	found = false
+	self.sttsEntryIndex = 0
+	for self.sttsEntryIndex < len(self.sample.TimeToSample.Entries) {
+		entry := self.sample.TimeToSample.Entries[self.sttsEntryIndex]
+		n := int(entry.Count)
+		if index >= start && index < start+n {
+			self.sampleIndexInSttsEntry = index - start
+			self.dts += int64(index-start) * int64(entry.Duration)
+			found = true
+			break
+		}
+		start += n
+		self.dts += int64(n) * int64(entry.Duration)
+		self.sttsEntryIndex++
+	}
+	if !found {
+		err = fmt.Errorf("mp4: stream[%d]: cannot locate sample index in stts entry", self.idx)
+		return
+	}
+
+	if self.sample.CompositionOffset != nil && len(self.sample.CompositionOffset.Entries) > 0 {
+		start = 0
+		found = false
+		self.cttsEntryIndex = 0
+		for self.cttsEntryIndex < len(self.sample.CompositionOffset.Entries) {
+			n := int(self.sample.CompositionOffset.Entries[self.cttsEntryIndex].Count)
+			if index >= start && index < start+n {
+				self.sampleIndexInCttsEntry = index - start
+				found = true
+				break
+			}
+			start += n
+			self.cttsEntryIndex++
+		}
+		if !found {
+			err = fmt.Errorf("mp4: stream[%d]: cannot locate sample index in ctts entry", self.idx)
+			return
+		}
+	}
+
+	if self.sample.SyncSample != nil {
+		self.syncSampleIndex = 0
+		for self.syncSampleIndex < len(self.sample.SyncSample.Entries)-1 {
+			if self.sample.SyncSample.Entries[self.syncSampleIndex+1]-1 > uint32(index) {
+				break
+			}
+			self.syncSampleIndex++
+		}
+	}
+
+	if false {
+		fmt.Printf("mp4: stream[%d]: setSampleIndex chunkGroupIndex=%d chunkIndex=%d sampleOffsetInChunk=%d\n",
+			self.idx, self.chunkGroupIndex, self.chunkIndex, self.sampleOffsetInChunk)
+	}
+
+	self.sampleIndex = index
+	return
+}
+
+func (self *Stream) isSampleValid() bool {
+	if self.chunkIndex >= len(self.sample.ChunkOffset.Entries) {
+		return false
+	}
+	if self.chunkGroupIndex >= len(self.sample.SampleToChunk.Entries) {
+		return false
+	}
+	if self.sttsEntryIndex >= len(self.sample.TimeToSample.Entries) {
+		return false
+	}
+	if self.sample.CompositionOffset != nil && len(self.sample.CompositionOffset.Entries) > 0 {
+		if self.cttsEntryIndex >= len(self.sample.CompositionOffset.Entries) {
+			return false
+		}
+	}
+	if self.sample.SyncSample != nil {
+		if self.syncSampleIndex >= len(self.sample.SyncSample.Entries) {
+			return false
+		}
+	}
+	if self.sample.SampleSize.SampleSize != 0 {
+		if self.sampleIndex >= len(self.sample.SampleSize.Entries) {
+			return false
+		}
+	}
+	return true
+}
+
+func (self *Stream) incSampleIndex() (duration int64) {
+	if false {
+		fmt.Printf("incSampleIndex sampleIndex=%d sampleOffsetInChunk=%d sampleIndexInChunk=%d chunkGroupIndex=%d chunkIndex=%d\n",
+			self.sampleIndex, self.sampleOffsetInChunk, self.sampleIndexInChunk, self.chunkGroupIndex, self.chunkIndex)
+	}
+
+	self.sampleIndexInChunk++
+	if uint32(self.sampleIndexInChunk) == self.sample.SampleToChunk.Entries[self.chunkGroupIndex].SamplesPerChunk {
+		self.chunkIndex++
+		self.sampleIndexInChunk = 0
+		self.sampleOffsetInChunk = int64(0)
+	} else {
+		if self.sample.SampleSize.SampleSize != 0 {
+			self.sampleOffsetInChunk += int64(self.sample.SampleSize.SampleSize)
+		} else {
+			self.sampleOffsetInChunk += int64(self.sample.SampleSize.Entries[self.sampleIndex])
+		}
+	}
+
+	if self.chunkGroupIndex+1 < len(self.sample.SampleToChunk.Entries) &&
+		uint32(self.chunkIndex+1) == self.sample.SampleToChunk.Entries[self.chunkGroupIndex+1].FirstChunk {
+		self.chunkGroupIndex++
+	}
+
+	sttsEntry := self.sample.TimeToSample.Entries[self.sttsEntryIndex]
+	duration = int64(sttsEntry.Duration)
+	self.sampleIndexInSttsEntry++
+	self.dts += duration
+	if uint32(self.sampleIndexInSttsEntry) == sttsEntry.Count {
+		self.sampleIndexInSttsEntry = 0
+		self.sttsEntryIndex++
+	}
+
+	if self.sample.CompositionOffset != nil && len(self.sample.CompositionOffset.Entries) > 0 {
+		self.sampleIndexInCttsEntry++
+		if uint32(self.sampleIndexInCttsEntry) == self.sample.CompositionOffset.Entries[self.cttsEntryIndex].Count {
+			self.sampleIndexInCttsEntry = 0
+			self.cttsEntryIndex++
+		}
+	}
+
+	if self.sample.SyncSample != nil {
+		entries := self.sample.SyncSample.Entries
+		if self.syncSampleIndex+1 < len(entries) && entries[self.syncSampleIndex+1]-1 == uint32(self.sampleIndex+1) {
+			self.syncSampleIndex++
+		}
+	}
+
+	self.sampleIndex++
+	return
+}
+
+func (self *Stream) sampleCount() int {
+	if self.sample.SampleSize.SampleSize == 0 {
+		chunkGroupIndex := 0
+		count := 0
+		for chunkIndex := range self.sample.ChunkOffset.Entries {
+			n := int(self.sample.SampleToChunk.Entries[chunkGroupIndex].SamplesPerChunk)
+			count += n
+			if chunkGroupIndex+1 < len(self.sample.SampleToChunk.Entries) &&
+				uint32(chunkIndex+1) == self.sample.SampleToChunk.Entries[chunkGroupIndex+1].FirstChunk {
+				chunkGroupIndex++
+			}
+		}
+		return count
+	} else {
+		return len(self.sample.SampleSize.Entries)
+	}
+}
+
+func (self *Demuxer) ReadPacket() (pkt av.Packet, err error) {
+	if err = self.probe(); err != nil {
+		return
+	}
+	if len(self.streams) == 0 {
+		err = errors.New("mp4: no streams available while trying to read a packet")
+		return
+	}
+
+	var chosen *Stream
+	var chosenidx int
+	for i, stream := range self.streams {
+		if chosen == nil || stream.tsToTime(stream.dts) < chosen.tsToTime(chosen.dts) {
+			chosen = stream
+			chosenidx = i
+		}
+	}
+	if false {
+		fmt.Printf("ReadPacket: chosen index=%v time=%v\n", chosen.idx, chosen.tsToTime(chosen.dts))
+	}
+	tm := chosen.tsToTime(chosen.dts)
+	if pkt, err = chosen.readPacket(); err != nil {
+		return
+	}
+	pkt.Time = tm
+	pkt.Idx = int8(chosenidx)
+	return
+}
+
+func (self *Demuxer) CurrentTime() (tm time.Duration) {
+	if len(self.streams) > 0 {
+		stream := self.streams[0]
+		tm = stream.tsToTime(stream.dts)
+	}
+	return
+}
+
+func (self *Demuxer) SeekToTime(tm time.Duration) (err error) {
+	for _, stream := range self.streams {
+		if stream.Type().IsVideo() {
+			if err = stream.seekToTime(tm); err != nil {
+				return
+			}
+			tm = stream.tsToTime(stream.dts)
+			break
+		}
+	}
+
+	for _, stream := range self.streams {
+		if !stream.Type().IsVideo() {
+			if err = stream.seekToTime(tm); err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func (self *Stream) readPacket() (pkt av.Packet, err error) {
+	if !self.isSampleValid() {
+		err = io.EOF
+		return
+	}
+	//fmt.Println("readPacket", self.sampleIndex)
+
+	chunkOffset := self.sample.ChunkOffset.Entries[self.chunkIndex]
+	sampleSize := uint32(0)
+	if self.sample.SampleSize.SampleSize != 0 {
+		sampleSize = self.sample.SampleSize.SampleSize
+	} else {
+		sampleSize = self.sample.SampleSize.Entries[self.sampleIndex]
+	}
+
+	sampleOffset := int64(chunkOffset) + self.sampleOffsetInChunk
+	pkt.Data = make([]byte, sampleSize)
+	if err = self.demuxer.readat(sampleOffset, pkt.Data); err != nil {
+		return
+	}
+
+	if self.sample.SyncSample != nil {
+		if self.sample.SyncSample.Entries[self.syncSampleIndex]-1 == uint32(self.sampleIndex) {
+			pkt.IsKeyFrame = true
+		}
+	}
+
+	//println("pts/dts", self.ptsEntryIndex, self.dtsEntryIndex)
+	if self.sample.CompositionOffset != nil && len(self.sample.CompositionOffset.Entries) > 0 {
+		cts := int64(self.sample.CompositionOffset.Entries[self.cttsEntryIndex].Offset)
+		pkt.CompositionTime = self.tsToTime(cts)
+	}
+
+	self.incSampleIndex()
+
+	return
+}
+
+func (self *Stream) seekToTime(tm time.Duration) (err error) {
+	index := self.timeToSampleIndex(tm)
+	if err = self.setSampleIndex(index); err != nil {
+		return
+	}
+	if false {
+		fmt.Printf("stream[%d]: seekToTime index=%v time=%v cur=%v\n", self.idx, index, tm, self.tsToTime(self.dts))
+	}
+	return
+}
+
+func (self *Stream) timeToSampleIndex(tm time.Duration) int {
+	targetTs := self.timeToTs(tm)
+	targetIndex := 0
+
+	startTs := int64(0)
+	endTs := int64(0)
+	startIndex := 0
+	endIndex := 0
+	found := false
+	for _, entry := range self.sample.TimeToSample.Entries {
+		endTs = startTs + int64(entry.Count*entry.Duration)
+		endIndex = startIndex + int(entry.Count)
+		if targetTs >= startTs && targetTs < endTs {
+			targetIndex = startIndex + int((targetTs-startTs)/int64(entry.Duration))
+			found = true
+		}
+		startTs = endTs
+		startIndex = endIndex
+	}
+	if !found {
+		if targetTs < 0 {
+			targetIndex = 0
+		} else {
+			targetIndex = endIndex - 1
+		}
+	}
+
+	if self.sample.SyncSample != nil {
+		entries := self.sample.SyncSample.Entries
+		for i := len(entries) - 1; i >= 0; i-- {
+			if entries[i]-1 < uint32(targetIndex) {
+				targetIndex = int(entries[i] - 1)
+				break
+			}
+		}
+	}
+
+	return targetIndex
+}

format/mp4/handler.go

@@ -0,0 +1,32 @@
+package mp4
+
+import (
+	"io"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/av/avutil"
+)
+
+var CodecTypes = []av.CodecType{av.H264, av.AAC}
+
+func Handler(h *avutil.RegisterHandler) {
+	h.Ext = ".mp4"
+
+	h.Probe = func(b []byte) bool {
+		switch string(b[4:8]) {
+		case "moov", "ftyp", "free", "mdat", "moof":
+			return true
+		}
+		return false
+	}
+
+	h.ReaderDemuxer = func(r io.Reader) av.Demuxer {
+		return NewDemuxer(r.(io.ReadSeeker))
+	}
+
+	h.WriterMuxer = func(w io.Writer) av.Muxer {
+		return NewMuxer(w.(io.WriteSeeker))
+	}
+
+	h.CodecTypes = CodecTypes
+}

format/mp4/mp4io/gen/pattern.go

@@ -0,0 +1,437 @@
+package main
+
+func moov_Movie() {
+	atom(Header, MovieHeader)
+	atom(MovieExtend, MovieExtend)
+	atoms(Tracks, Track)
+	_unknowns()
+}
+
+func mvhd_MovieHeader() {
+	uint8(Version)
+	uint24(Flags)
+	time32(CreateTime)
+	time32(ModifyTime)
+	int32(TimeScale)
+	int32(Duration)
+	fixed32(PreferredRate)
+	fixed16(PreferredVolume)
+	_skip(10)
+	array(Matrix, int32, 9)
+	time32(PreviewTime)
+	time32(PreviewDuration)
+	time32(PosterTime)
+	time32(SelectionTime)
+	time32(SelectionDuration)
+	time32(CurrentTime)
+	int32(NextTrackId)
+}
+
+func trak_Track() {
+	atom(Header, TrackHeader)
+	atom(Media, Media)
+	_unknowns()
+}
+
+func tkhd_TrackHeader() {
+	uint8(Version)
+	uint24(Flags)
+	time32(CreateTime)
+	time32(ModifyTime)
+	int32(TrackId)
+	_skip(4)
+	int32(Duration)
+	_skip(8)
+	int16(Layer)
+	int16(AlternateGroup)
+	fixed16(Volume)
+	_skip(2)
+	array(Matrix, int32, 9)
+	fixed32(TrackWidth)
+	fixed32(TrackHeight)
+}
+
+func hdlr_HandlerRefer() {
+	uint8(Version)
+	uint24(Flags)
+	bytes(Type, 4)
+	bytes(SubType, 4)
+	bytesleft(Name)
+}
+
+func mdia_Media() {
+	atom(Header, MediaHeader)
+	atom(Handler, HandlerRefer)
+	atom(Info, MediaInfo)
+	_unknowns()
+}
+
+func mdhd_MediaHeader() {
+	uint8(Version)
+	uint24(Flags)
+	time32(CreateTime)
+	time32(ModifyTime)
+	int32(TimeScale)
+	int32(Duration)
+	int16(Language)
+	int16(Quality)
+}
+
+func minf_MediaInfo() {
+	atom(Sound, SoundMediaInfo)
+	atom(Video, VideoMediaInfo)
+	atom(Data, DataInfo)
+	atom(Sample, SampleTable)
+	_unknowns()
+}
+
+func dinf_DataInfo() {
+	atom(Refer, DataRefer)
+	_unknowns()
+}
+
+func dref_DataRefer() {
+	uint8(Version)
+	uint24(Flags)
+	int32(_childrenNR)
+	atom(Url, DataReferUrl)
+}
+
+func url__DataReferUrl() {
+	uint8(Version)
+	uint24(Flags)
+}
+
+func smhd_SoundMediaInfo() {
+	uint8(Version)
+	uint24(Flags)
+	int16(Balance)
+	_skip(2)
+}
+
+func vmhd_VideoMediaInfo() {
+	uint8(Version)
+	uint24(Flags)
+	int16(GraphicsMode)
+	array(Opcolor, int16, 3)
+}
+
+func stbl_SampleTable() {
+	atom(SampleDesc, SampleDesc)
+	atom(TimeToSample, TimeToSample)
+	atom(CompositionOffset, CompositionOffset)
+	atom(SampleToChunk, SampleToChunk)
+	atom(SyncSample, SyncSample)
+	atom(ChunkOffset, ChunkOffset)
+	atom(SampleSize, SampleSize)
+}
+
+func stsd_SampleDesc() {
+	uint8(Version)
+	_skip(3)
+	int32(_childrenNR)
+	atom(AVC1Desc, AVC1Desc)
+	atom(MP4ADesc, MP4ADesc)
+	_unknowns()
+}
+
+func mp4a_MP4ADesc() {
+	_skip(6)
+	int16(DataRefIdx)
+	int16(Version)
+	int16(RevisionLevel)
+	int32(Vendor)
+	int16(NumberOfChannels)
+	int16(SampleSize)
+	int16(CompressionId)
+	_skip(2)
+	fixed32(SampleRate)
+	atom(Conf, ElemStreamDesc)
+	_unknowns()
+}
+
+func avc1_AVC1Desc() {
+	_skip(6)
+	int16(DataRefIdx)
+	int16(Version)
+	int16(Revision)
+	int32(Vendor)
+	int32(TemporalQuality)
+	int32(SpatialQuality)
+	int16(Width)
+	int16(Height)
+	fixed32(HorizontalResolution)
+	fixed32(VorizontalResolution)
+	_skip(4)
+	int16(FrameCount)
+	bytes(CompressorName, 32)
+	int16(Depth)
+	int16(ColorTableId)
+	atom(Conf, AVC1Conf)
+	_unknowns()
+}
+
+func avcC_AVC1Conf() {
+	bytesleft(Data)
+}
+
+func stts_TimeToSample() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, TimeToSampleEntry)
+}
+
+func TimeToSampleEntry() {
+	uint32(Count)
+	uint32(Duration)
+}
+
+func stsc_SampleToChunk() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, SampleToChunkEntry)
+}
+
+func SampleToChunkEntry() {
+	uint32(FirstChunk)
+	uint32(SamplesPerChunk)
+	uint32(SampleDescId)
+}
+
+func ctts_CompositionOffset() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, CompositionOffsetEntry)
+}
+
+func CompositionOffsetEntry() {
+	uint32(Count)
+	uint32(Offset)
+}
+
+func stss_SyncSample() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, uint32)
+}
+
+func stco_ChunkOffset() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, uint32)
+}
+
+func moof_MovieFrag() {
+	atom(Header, MovieFragHeader)
+	atoms(Tracks, TrackFrag)
+	_unknowns()
+}
+
+func mfhd_MovieFragHeader() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(Seqnum)
+}
+
+func traf_TrackFrag() {
+	atom(Header, TrackFragHeader)
+	atom(DecodeTime, TrackFragDecodeTime)
+	atom(Run, TrackFragRun)
+	_unknowns()
+}
+
+func mvex_MovieExtend() {
+	atoms(Tracks, TrackExtend)
+	_unknowns()
+}
+
+func trex_TrackExtend() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(TrackId)
+	uint32(DefaultSampleDescIdx)
+	uint32(DefaultSampleDuration)
+	uint32(DefaultSampleSize)
+	uint32(DefaultSampleFlags)
+}
+
+func stsz_SampleSize() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(SampleSize)
+	_code(func() {
+		if self.SampleSize != 0 {
+			return
+		}
+	})
+	uint32(_len_Entries)
+	slice(Entries, uint32)
+}
+
+func trun_TrackFragRun() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+
+	uint32(DataOffset, _code(func() {
+		if self.Flags&TRUN_DATA_OFFSET != 0 {
+			doit()
+		}
+	}))
+
+	uint32(FirstSampleFlags, _code(func() {
+		if self.Flags&TRUN_FIRST_SAMPLE_FLAGS != 0 {
+			doit()
+		}
+	}))
+
+	slice(Entries, TrackFragRunEntry, _code(func() {
+		for i, entry := range self.Entries {
+			var flags uint32
+			if i > 0 {
+				flags = self.Flags
+			} else {
+				flags = self.FirstSampleFlags
+			}
+			if flags&TRUN_SAMPLE_DURATION != 0 {
+				pio.PutU32BE(b[n:], entry.Duration)
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_SIZE != 0 {
+				pio.PutU32BE(b[n:], entry.Size)
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_FLAGS != 0 {
+				pio.PutU32BE(b[n:], entry.Flags)
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_CTS != 0 {
+				pio.PutU32BE(b[n:], entry.Cts)
+				n += 4
+			}
+		}
+	}, func() {
+		for i := range self.Entries {
+			var flags uint32
+			if i > 0 {
+				flags = self.Flags
+			} else {
+				flags = self.FirstSampleFlags
+			}
+			if flags&TRUN_SAMPLE_DURATION != 0 {
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_SIZE != 0 {
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_FLAGS != 0 {
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_CTS != 0 {
+				n += 4
+			}
+		}
+	}, func() {
+		for i := 0; i < int(_len_Entries); i++ {
+			var flags uint32
+			if i > 0 {
+				flags = self.Flags
+			} else {
+				flags = self.FirstSampleFlags
+			}
+			entry := &self.Entries[i]
+			if flags&TRUN_SAMPLE_DURATION != 0 {
+				entry.Duration = pio.U32BE(b[n:])
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_SIZE != 0 {
+				entry.Size = pio.U32BE(b[n:])
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_FLAGS != 0 {
+				entry.Flags = pio.U32BE(b[n:])
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_CTS != 0 {
+				entry.Cts = pio.U32BE(b[n:])
+				n += 4
+			}
+		}
+	}))
+}
+
+func TrackFragRunEntry() {
+	uint32(Duration)
+	uint32(Size)
+	uint32(Flags)
+	uint32(Cts)
+}
+
+func tfhd_TrackFragHeader() {
+	uint8(Version)
+	uint24(Flags)
+
+	uint64(BaseDataOffset, _code(func() {
+		if self.Flags&TFHD_BASE_DATA_OFFSET != 0 {
+			doit()
+		}
+	}))
+
+	uint32(StsdId, _code(func() {
+		if self.Flags&TFHD_STSD_ID != 0 {
+			doit()
+		}
+	}))
+
+	uint32(DefaultDuration, _code(func() {
+		if self.Flags&TFHD_DEFAULT_DURATION != 0 {
+			doit()
+		}
+	}))
+
+	uint32(DefaultSize, _code(func() {
+		if self.Flags&TFHD_DEFAULT_SIZE != 0 {
+			doit()
+		}
+	}))
+
+	uint32(DefaultFlags, _code(func() {
+		if self.Flags&TFHD_DEFAULT_FLAGS != 0 {
+			doit()
+		}
+	}))
+}
+
+func tfdt_TrackFragDecodeTime() {
+	uint8(Version)
+	uint24(Flags)
+	time64(Time, _code(func() {
+		if self.Version != 0 {
+			PutTime64(b[n:], self.Time)
+			n += 8
+		} else {
+			PutTime32(b[n:], self.Time)
+			n += 4
+		}
+	}, func() {
+		if self.Version != 0 {
+			n += 8
+		} else {
+			n += 4
+		}
+	}, func() {
+		if self.Version != 0 {
+			self.Time = GetTime64(b[n:])
+			n += 8
+		} else {
+			self.Time = GetTime32(b[n:])
+			n += 4
+		}
+	}))
+}
+

format/mp4/mp4io/mp4io.go

@@ -0,0 +1,502 @@
+package mp4io
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+type ParseError struct {
+	Debug  string
+	Offset int
+	prev   *ParseError
+}
+
+func (self *ParseError) Error() string {
+	s := []string{}
+	for p := self; p != nil; p = p.prev {
+		s = append(s, fmt.Sprintf("%s:%d", p.Debug, p.Offset))
+	}
+	return "mp4io: parse error: " + strings.Join(s, ",")
+}
+
+func parseErr(debug string, offset int, prev error) (err error) {
+	_prev, _ := prev.(*ParseError)
+	return &ParseError{Debug: debug, Offset: offset, prev: _prev}
+}
+
+func GetTime32(b []byte) (t time.Time) {
+	sec := pio.U32BE(b)
+	t = time.Date(1904, time.January, 1, 0, 0, 0, 0, time.UTC)
+	t = t.Add(time.Second * time.Duration(sec))
+	return
+}
+
+func PutTime32(b []byte, t time.Time) {
+	dur := t.Sub(time.Date(1904, time.January, 1, 0, 0, 0, 0, time.UTC))
+	sec := uint32(dur / time.Second)
+	pio.PutU32BE(b, sec)
+}
+
+func GetTime64(b []byte) (t time.Time) {
+	sec := pio.U64BE(b)
+	t = time.Date(1904, time.January, 1, 0, 0, 0, 0, time.UTC)
+	t = t.Add(time.Second * time.Duration(sec))
+	return
+}
+
+func PutTime64(b []byte, t time.Time) {
+	dur := t.Sub(time.Date(1904, time.January, 1, 0, 0, 0, 0, time.UTC))
+	sec := uint64(dur / time.Second)
+	pio.PutU64BE(b, sec)
+}
+
+func PutFixed16(b []byte, f float64) {
+	intpart, fracpart := math.Modf(f)
+	b[0] = uint8(intpart)
+	b[1] = uint8(fracpart * 256.0)
+}
+
+func GetFixed16(b []byte) float64 {
+	return float64(b[0]) + float64(b[1])/256.0
+}
+
+func PutFixed32(b []byte, f float64) {
+	intpart, fracpart := math.Modf(f)
+	pio.PutU16BE(b[0:2], uint16(intpart))
+	pio.PutU16BE(b[2:4], uint16(fracpart*65536.0))
+}
+
+func GetFixed32(b []byte) float64 {
+	return float64(pio.U16BE(b[0:2])) + float64(pio.U16BE(b[2:4]))/65536.0
+}
+
+type Tag uint32
+
+func (self Tag) String() string {
+	var b [4]byte
+	pio.PutU32BE(b[:], uint32(self))
+	for i := 0; i < 4; i++ {
+		if b[i] == 0 {
+			b[i] = ' '
+		}
+	}
+	return string(b[:])
+}
+
+type Atom interface {
+	Pos() (int, int)
+	Tag() Tag
+	Marshal([]byte) int
+	Unmarshal([]byte, int) (int, error)
+	Len() int
+	Children() []Atom
+}
+
+type AtomPos struct {
+	Offset int
+	Size   int
+}
+
+func (self AtomPos) Pos() (int, int) {
+	return self.Offset, self.Size
+}
+
+func (self *AtomPos) setPos(offset int, size int) {
+	self.Offset, self.Size = offset, size
+}
+
+type Dummy struct {
+	Data []byte
+	Tag_ Tag
+	AtomPos
+}
+
+func (self Dummy) Children() []Atom {
+	return nil
+}
+
+func (self Dummy) Tag() Tag {
+	return self.Tag_
+}
+
+func (self Dummy) Len() int {
+	return len(self.Data)
+}
+
+func (self Dummy) Marshal(b []byte) int {
+	copy(b, self.Data)
+	return len(self.Data)
+}
+
+func (self *Dummy) Unmarshal(b []byte, offset int) (n int, err error) {
+	(&self.AtomPos).setPos(offset, len(b))
+	self.Data = b
+	n = len(b)
+	return
+}
+
+func StringToTag(tag string) Tag {
+	var b [4]byte
+	copy(b[:], []byte(tag))
+	return Tag(pio.U32BE(b[:]))
+}
+
+func FindChildrenByName(root Atom, tag string) Atom {
+	return FindChildren(root, StringToTag(tag))
+}
+
+func FindChildren(root Atom, tag Tag) Atom {
+	if root.Tag() == tag {
+		return root
+	}
+	for _, child := range root.Children() {
+		if r := FindChildren(child, tag); r != nil {
+			return r
+		}
+	}
+	return nil
+}
+
+const (
+	TFHD_BASE_DATA_OFFSET     = 0x01
+	TFHD_STSD_ID              = 0x02
+	TFHD_DEFAULT_DURATION     = 0x08
+	TFHD_DEFAULT_SIZE         = 0x10
+	TFHD_DEFAULT_FLAGS        = 0x20
+	TFHD_DURATION_IS_EMPTY    = 0x010000
+	TFHD_DEFAULT_BASE_IS_MOOF = 0x020000
+)
+
+const (
+	TRUN_DATA_OFFSET        = 0x01
+	TRUN_FIRST_SAMPLE_FLAGS = 0x04
+	TRUN_SAMPLE_DURATION    = 0x100
+	TRUN_SAMPLE_SIZE        = 0x200
+	TRUN_SAMPLE_FLAGS       = 0x400
+	TRUN_SAMPLE_CTS         = 0x800
+)
+
+const (
+	MP4ESDescrTag          = 3
+	MP4DecConfigDescrTag   = 4
+	MP4DecSpecificDescrTag = 5
+)
+
+type ElemStreamDesc struct {
+	DecConfig []byte
+	TrackId   uint16
+	AtomPos
+}
+
+func (self ElemStreamDesc) Children() []Atom {
+	return nil
+}
+
+func (self ElemStreamDesc) fillLength(b []byte, length int) (n int) {
+	for i := 3; i > 0; i-- {
+		b[n] = uint8(length>>uint(7*i))&0x7f | 0x80
+		n++
+	}
+	b[n] = uint8(length & 0x7f)
+	n++
+	return
+}
+
+func (self ElemStreamDesc) lenDescHdr() (n int) {
+	return 5
+}
+
+func (self ElemStreamDesc) fillDescHdr(b []byte, tag uint8, datalen int) (n int) {
+	b[n] = tag
+	n++
+	n += self.fillLength(b[n:], datalen)
+	return
+}
+
+func (self ElemStreamDesc) lenESDescHdr() (n int) {
+	return self.lenDescHdr() + 3
+}
+
+func (self ElemStreamDesc) fillESDescHdr(b []byte, datalen int) (n int) {
+	n += self.fillDescHdr(b[n:], MP4ESDescrTag, datalen)
+	pio.PutU16BE(b[n:], self.TrackId)
+	n += 2
+	b[n] = 0 // flags
+	n++
+	return
+}
+
+func (self ElemStreamDesc) lenDecConfigDescHdr() (n int) {
+	return self.lenDescHdr() + 2 + 3 + 4 + 4 + self.lenDescHdr()
+}
+
+func (self ElemStreamDesc) fillDecConfigDescHdr(b []byte, datalen int) (n int) {
+	n += self.fillDescHdr(b[n:], MP4DecConfigDescrTag, datalen)
+	b[n] = 0x40 // objectid
+	n++
+	b[n] = 0x15 // streamtype
+	n++
+	// buffer size db
+	pio.PutU24BE(b[n:], 0)
+	n += 3
+	// max bitrage
+	pio.PutU32BE(b[n:], uint32(200000))
+	n += 4
+	// avg bitrage
+	pio.PutU32BE(b[n:], uint32(0))
+	n += 4
+	n += self.fillDescHdr(b[n:], MP4DecSpecificDescrTag, datalen-n)
+	return
+}
+
+func (self ElemStreamDesc) Len() (n int) {
+	return 8 + 4 + self.lenESDescHdr() + self.lenDecConfigDescHdr() + len(self.DecConfig) + self.lenDescHdr() + 1
+}
+
+// Version(4)
+// ESDesc(
+//   MP4ESDescrTag
+//   ESID(2)
+//   ESFlags(1)
+//   DecConfigDesc(
+//     MP4DecConfigDescrTag
+//     objectId streamType bufSize avgBitrate
+//     DecSpecificDesc(
+//       MP4DecSpecificDescrTag
+//       decConfig
+//     )
+//   )
+//   ?Desc(lenDescHdr+1)
+// )
+
+func (self ElemStreamDesc) Marshal(b []byte) (n int) {
+	pio.PutU32BE(b[4:], uint32(ESDS))
+	n += 8
+	pio.PutU32BE(b[n:], 0) // Version
+	n += 4
+	datalen := self.Len()
+	n += self.fillESDescHdr(b[n:], datalen-n-self.lenESDescHdr())
+	n += self.fillDecConfigDescHdr(b[n:], datalen-n-self.lenDescHdr()-1)
+	copy(b[n:], self.DecConfig)
+	n += len(self.DecConfig)
+	n += self.fillDescHdr(b[n:], 0x06, datalen-n-self.lenDescHdr())
+	b[n] = 0x02
+	n++
+	pio.PutU32BE(b[0:], uint32(n))
+	return
+}
+
+func (self *ElemStreamDesc) Unmarshal(b []byte, offset int) (n int, err error) {
+	if len(b) < n+12 {
+		err = parseErr("hdr", offset+n, err)
+		return
+	}
+	(&self.AtomPos).setPos(offset, len(b))
+	n += 8
+	n += 4
+	return self.parseDesc(b[n:], offset+n)
+}
+
+func (self *ElemStreamDesc) parseDesc(b []byte, offset int) (n int, err error) {
+	var hdrlen int
+	var datalen int
+	var tag uint8
+	if hdrlen, tag, datalen, err = self.parseDescHdr(b, offset); err != nil {
+		return
+	}
+	n += hdrlen
+
+	if len(b) < n+datalen {
+		err = parseErr("datalen", offset+n, err)
+		return
+	}
+
+	switch tag {
+	case MP4ESDescrTag:
+		if len(b) < n+3 {
+			err = parseErr("MP4ESDescrTag", offset+n, err)
+			return
+		}
+		if _, err = self.parseDesc(b[n+3:], offset+n+3); err != nil {
+			return
+		}
+
+	case MP4DecConfigDescrTag:
+		const size = 2 + 3 + 4 + 4
+		if len(b) < n+size {
+			err = parseErr("MP4DecSpecificDescrTag", offset+n, err)
+			return
+		}
+		if _, err = self.parseDesc(b[n+size:], offset+n+size); err != nil {
+			return
+		}
+
+	case MP4DecSpecificDescrTag:
+		self.DecConfig = b[n:]
+	}
+
+	n += datalen
+	return
+}
+
+func (self *ElemStreamDesc) parseLength(b []byte, offset int) (n int, length int, err error) {
+	for n < 4 {
+		if len(b) < n+1 {
+			err = parseErr("len", offset+n, err)
+			return
+		}
+		c := b[n]
+		n++
+		length = (length << 7) | (int(c) & 0x7f)
+		if c&0x80 == 0 {
+			break
+		}
+	}
+	return
+}
+
+func (self *ElemStreamDesc) parseDescHdr(b []byte, offset int) (n int, tag uint8, datalen int, err error) {
+	if len(b) < n+1 {
+		err = parseErr("tag", offset+n, err)
+		return
+	}
+	tag = b[n]
+	n++
+	var lenlen int
+	if lenlen, datalen, err = self.parseLength(b[n:], offset+n); err != nil {
+		return
+	}
+	n += lenlen
+	return
+}
+
+func ReadFileAtoms(r io.ReadSeeker) (atoms []Atom, err error) {
+	for {
+		offset, _ := r.Seek(0, 1)
+		taghdr := make([]byte, 8)
+		if _, err = io.ReadFull(r, taghdr); err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			return
+		}
+		size := pio.U32BE(taghdr[0:])
+		tag := Tag(pio.U32BE(taghdr[4:]))
+
+		var atom Atom
+		switch tag {
+		case MOOV:
+			atom = &Movie{}
+		case MOOF:
+			atom = &MovieFrag{}
+		}
+
+		if atom != nil {
+			b := make([]byte, int(size))
+			if _, err = io.ReadFull(r, b[8:]); err != nil {
+				return
+			}
+			copy(b, taghdr)
+			if _, err = atom.Unmarshal(b, int(offset)); err != nil {
+				return
+			}
+			atoms = append(atoms, atom)
+		} else {
+			dummy := &Dummy{Tag_: tag}
+			dummy.setPos(int(offset), int(size))
+			if _, err = r.Seek(int64(size)-8, 1); err != nil {
+				return
+			}
+			atoms = append(atoms, dummy)
+		}
+	}
+	return
+}
+
+func printatom(out io.Writer, root Atom, depth int) {
+	offset, size := root.Pos()
+
+	type stringintf interface {
+		String() string
+	}
+
+	fmt.Fprintf(out,
+		"%s%s offset=%d size=%d",
+		strings.Repeat(" ", depth*2), root.Tag(), offset, size,
+	)
+	if str, ok := root.(stringintf); ok {
+		fmt.Fprint(out, " ", str.String())
+	}
+	fmt.Fprintln(out)
+
+	children := root.Children()
+	for _, child := range children {
+		printatom(out, child, depth+1)
+	}
+}
+
+func FprintAtom(out io.Writer, root Atom) {
+	printatom(out, root, 0)
+}
+
+func PrintAtom(root Atom) {
+	FprintAtom(os.Stdout, root)
+}
+
+func (self MovieHeader) String() string {
+	return fmt.Sprintf("dur=%d", self.Duration)
+}
+
+func (self TimeToSample) String() string {
+	return fmt.Sprintf("entries=%d", len(self.Entries))
+}
+
+func (self SampleToChunk) String() string {
+	return fmt.Sprintf("entries=%d", len(self.Entries))
+}
+
+func (self SampleSize) String() string {
+	return fmt.Sprintf("entries=%d", len(self.Entries))
+}
+
+func (self SyncSample) String() string {
+	return fmt.Sprintf("entries=%d", len(self.Entries))
+}
+
+func (self CompositionOffset) String() string {
+	return fmt.Sprintf("entries=%d", len(self.Entries))
+}
+
+func (self ChunkOffset) String() string {
+	return fmt.Sprintf("entries=%d", len(self.Entries))
+}
+
+func (self TrackFragRun) String() string {
+	return fmt.Sprintf("dataoffset=%d", self.DataOffset)
+}
+
+func (self TrackFragHeader) String() string {
+	return fmt.Sprintf("basedataoffset=%d", self.BaseDataOffset)
+}
+
+func (self ElemStreamDesc) String() string {
+	return fmt.Sprintf("configlen=%d", len(self.DecConfig))
+}
+
+func (self *Track) GetAVC1Conf() (conf *AVC1Conf) {
+	atom := FindChildren(self, AVCC)
+	conf, _ = atom.(*AVC1Conf)
+	return
+}
+
+func (self *Track) GetElemStreamDesc() (esds *ElemStreamDesc) {
+	atom := FindChildren(self, ESDS)
+	esds, _ = atom.(*ElemStreamDesc)
+	return
+}

format/mp4/muxer.go

@@ -0,0 +1,283 @@
+package mp4
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/codec/aacparser"
+	"github.com/deepch/vdk/codec/h264parser"
+	"github.com/deepch/vdk/format/mp4/mp4io"
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+type Muxer struct {
+	w       io.WriteSeeker
+	bufw    *bufio.Writer
+	wpos    int64
+	streams []*Stream
+}
+
+func NewMuxer(w io.WriteSeeker) *Muxer {
+	return &Muxer{
+		w:    w,
+		bufw: bufio.NewWriterSize(w, pio.RecommendBufioSize),
+	}
+}
+
+func (self *Muxer) newStream(codec av.CodecData) (err error) {
+	switch codec.Type() {
+	case av.H264, av.AAC:
+
+	default:
+		err = fmt.Errorf("mp4: codec type=%v is not supported", codec.Type())
+		return
+	}
+	stream := &Stream{CodecData: codec}
+
+	stream.sample = &mp4io.SampleTable{
+		SampleDesc:   &mp4io.SampleDesc{},
+		TimeToSample: &mp4io.TimeToSample{},
+		SampleToChunk: &mp4io.SampleToChunk{
+			Entries: []mp4io.SampleToChunkEntry{
+				{
+					FirstChunk:      1,
+					SampleDescId:    1,
+					SamplesPerChunk: 1,
+				},
+			},
+		},
+		SampleSize:  &mp4io.SampleSize{},
+		ChunkOffset: &mp4io.ChunkOffset{},
+	}
+
+	stream.trackAtom = &mp4io.Track{
+		Header: &mp4io.TrackHeader{
+			TrackId:  int32(len(self.streams) + 1),
+			Flags:    0x0003, // Track enabled | Track in movie
+			Duration: 0,      // fill later
+			Matrix:   [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
+		},
+		Media: &mp4io.Media{
+			Header: &mp4io.MediaHeader{
+				TimeScale: 0, // fill later
+				Duration:  0, // fill later
+				Language:  21956,
+			},
+			Info: &mp4io.MediaInfo{
+				Sample: stream.sample,
+				Data: &mp4io.DataInfo{
+					Refer: &mp4io.DataRefer{
+						Url: &mp4io.DataReferUrl{
+							Flags: 0x000001, // Self reference
+						},
+					},
+				},
+			},
+		},
+	}
+
+	switch codec.Type() {
+	case av.H264:
+		stream.sample.SyncSample = &mp4io.SyncSample{}
+	}
+
+	stream.timeScale = 90000
+	stream.muxer = self
+	self.streams = append(self.streams, stream)
+
+	return
+}
+
+func (self *Stream) fillTrackAtom() (err error) {
+	self.trackAtom.Media.Header.TimeScale = int32(self.timeScale)
+	self.trackAtom.Media.Header.Duration = int32(self.duration)
+
+	if self.Type() == av.H264 {
+		codec := self.CodecData.(h264parser.CodecData)
+		width, height := codec.Width(), codec.Height()
+		self.sample.SampleDesc.AVC1Desc = &mp4io.AVC1Desc{
+			DataRefIdx:           1,
+			HorizontalResolution: 72,
+			VorizontalResolution: 72,
+			Width:                int16(width),
+			Height:               int16(height),
+			FrameCount:           1,
+			Depth:                24,
+			ColorTableId:         -1,
+			Conf:                 &mp4io.AVC1Conf{Data: codec.AVCDecoderConfRecordBytes()},
+		}
+		self.trackAtom.Media.Handler = &mp4io.HandlerRefer{
+			SubType: [4]byte{'v', 'i', 'd', 'e'},
+			Name:    []byte("Video Media Handler"),
+		}
+		self.trackAtom.Media.Info.Video = &mp4io.VideoMediaInfo{
+			Flags: 0x000001,
+		}
+		self.trackAtom.Header.TrackWidth = float64(width)
+		self.trackAtom.Header.TrackHeight = float64(height)
+
+	} else if self.Type() == av.AAC {
+		codec := self.CodecData.(aacparser.CodecData)
+		self.sample.SampleDesc.MP4ADesc = &mp4io.MP4ADesc{
+			DataRefIdx:       1,
+			NumberOfChannels: int16(codec.ChannelLayout().Count()),
+			SampleSize:       int16(codec.SampleFormat().BytesPerSample()),
+			SampleRate:       float64(codec.SampleRate()),
+			Conf: &mp4io.ElemStreamDesc{
+				DecConfig: codec.MPEG4AudioConfigBytes(),
+			},
+		}
+		self.trackAtom.Header.Volume = 1
+		self.trackAtom.Header.AlternateGroup = 1
+		self.trackAtom.Media.Handler = &mp4io.HandlerRefer{
+			SubType: [4]byte{'s', 'o', 'u', 'n'},
+			Name:    []byte("Sound Handler"),
+		}
+		self.trackAtom.Media.Info.Sound = &mp4io.SoundMediaInfo{}
+
+	} else {
+		err = fmt.Errorf("mp4: codec type=%d invalid", self.Type())
+	}
+
+	return
+}
+
+func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) {
+	self.streams = []*Stream{}
+	for _, stream := range streams {
+		if err = self.newStream(stream); err != nil {
+			return
+		}
+	}
+
+	taghdr := make([]byte, 8)
+	pio.PutU32BE(taghdr[4:], uint32(mp4io.MDAT))
+	if _, err = self.w.Write(taghdr); err != nil {
+		return
+	}
+	self.wpos += 8
+
+	for _, stream := range self.streams {
+		if stream.Type().IsVideo() {
+			stream.sample.CompositionOffset = &mp4io.CompositionOffset{}
+		}
+	}
+	return
+}
+
+func (self *Muxer) WritePacket(pkt av.Packet) (err error) {
+	stream := self.streams[pkt.Idx]
+	if stream.lastpkt != nil {
+		if err = stream.writePacket(*stream.lastpkt, pkt.Time-stream.lastpkt.Time); err != nil {
+			return
+		}
+	}
+	stream.lastpkt = &pkt
+	return
+}
+
+func (self *Stream) writePacket(pkt av.Packet, rawdur time.Duration) (err error) {
+	if rawdur < 0 {
+		err = fmt.Errorf("mp4: stream#%d time=%v < lasttime=%v", pkt.Idx, pkt.Time, self.lastpkt.Time)
+		return
+	}
+
+	if _, err = self.muxer.bufw.Write(pkt.Data); err != nil {
+		return
+	}
+
+	if pkt.IsKeyFrame && self.sample.SyncSample != nil {
+		self.sample.SyncSample.Entries = append(self.sample.SyncSample.Entries, uint32(self.sampleIndex+1))
+	}
+
+	duration := uint32(self.timeToTs(rawdur))
+	if self.sttsEntry == nil || duration != self.sttsEntry.Duration {
+		self.sample.TimeToSample.Entries = append(self.sample.TimeToSample.Entries, mp4io.TimeToSampleEntry{Duration: duration})
+		self.sttsEntry = &self.sample.TimeToSample.Entries[len(self.sample.TimeToSample.Entries)-1]
+	}
+	self.sttsEntry.Count++
+
+	if self.sample.CompositionOffset != nil {
+		offset := uint32(self.timeToTs(pkt.CompositionTime))
+		if self.cttsEntry == nil || offset != self.cttsEntry.Offset {
+			table := self.sample.CompositionOffset
+			table.Entries = append(table.Entries, mp4io.CompositionOffsetEntry{Offset: offset})
+			self.cttsEntry = &table.Entries[len(table.Entries)-1]
+		}
+		self.cttsEntry.Count++
+	}
+
+	self.duration += int64(duration)
+	self.sampleIndex++
+	self.sample.ChunkOffset.Entries = append(self.sample.ChunkOffset.Entries, uint32(self.muxer.wpos))
+	self.sample.SampleSize.Entries = append(self.sample.SampleSize.Entries, uint32(len(pkt.Data)))
+
+	self.muxer.wpos += int64(len(pkt.Data))
+	return
+}
+
+func (self *Muxer) WriteTrailer() (err error) {
+	for _, stream := range self.streams {
+		if stream.lastpkt != nil {
+			if err = stream.writePacket(*stream.lastpkt, 0); err != nil {
+				return
+			}
+			stream.lastpkt = nil
+		}
+	}
+
+	moov := &mp4io.Movie{}
+	moov.Header = &mp4io.MovieHeader{
+		PreferredRate:   1,
+		PreferredVolume: 1,
+		Matrix:          [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
+		NextTrackId:     2,
+	}
+
+	maxDur := time.Duration(0)
+	timeScale := int64(10000)
+	for _, stream := range self.streams {
+		if err = stream.fillTrackAtom(); err != nil {
+			return
+		}
+		dur := stream.tsToTime(stream.duration)
+		stream.trackAtom.Header.Duration = int32(timeToTs(dur, timeScale))
+		if dur > maxDur {
+			maxDur = dur
+		}
+		moov.Tracks = append(moov.Tracks, stream.trackAtom)
+	}
+	moov.Header.TimeScale = int32(timeScale)
+	moov.Header.Duration = int32(timeToTs(maxDur, timeScale))
+
+	if err = self.bufw.Flush(); err != nil {
+		return
+	}
+
+	var mdatsize int64
+	if mdatsize, err = self.w.Seek(0, 1); err != nil {
+		return
+	}
+	if _, err = self.w.Seek(0, 0); err != nil {
+		return
+	}
+	taghdr := make([]byte, 4)
+	pio.PutU32BE(taghdr, uint32(mdatsize))
+	if _, err = self.w.Write(taghdr); err != nil {
+		return
+	}
+
+	if _, err = self.w.Seek(0, 2); err != nil {
+		return
+	}
+	b := make([]byte, moov.Len())
+	moov.Marshal(b)
+	if _, err = self.w.Write(b); err != nil {
+		return
+	}
+
+	return
+}

format/mp4/stream.go

@@ -0,0 +1,59 @@
+package mp4
+
+import (
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/format/mp4/mp4io"
+)
+
+type Stream struct {
+	av.CodecData
+
+	trackAtom *mp4io.Track
+	idx       int
+
+	lastpkt *av.Packet
+
+	timeScale int64
+	duration  int64
+
+	muxer   *Muxer
+	demuxer *Demuxer
+
+	sample      *mp4io.SampleTable
+	sampleIndex int
+
+	sampleOffsetInChunk int64
+	syncSampleIndex     int
+
+	dts                    int64
+	sttsEntryIndex         int
+	sampleIndexInSttsEntry int
+
+	cttsEntryIndex         int
+	sampleIndexInCttsEntry int
+
+	chunkGroupIndex    int
+	chunkIndex         int
+	sampleIndexInChunk int
+
+	sttsEntry *mp4io.TimeToSampleEntry
+	cttsEntry *mp4io.CompositionOffsetEntry
+}
+
+func timeToTs(tm time.Duration, timeScale int64) int64 {
+	return int64(tm * time.Duration(timeScale) / time.Second)
+}
+
+func tsToTime(ts int64, timeScale int64) time.Duration {
+	return time.Duration(ts) * time.Second / time.Duration(timeScale)
+}
+
+func (self *Stream) timeToTs(tm time.Duration) int64 {
+	return int64(tm * time.Duration(self.timeScale) / time.Second)
+}
+
+func (self *Stream) tsToTime(ts int64) time.Duration {
+	return time.Duration(ts) * time.Second / time.Duration(self.timeScale)
+}

format/mp4f/fd.go

@@ -0,0 +1,30 @@
+package mp4f
+
+import "github.com/deepch/vdk/format/mp4/mp4io"
+
+type FDummy struct {
+	Data []byte
+	Tag_ mp4io.Tag
+	mp4io.AtomPos
+}
+
+func (self FDummy) Children() []mp4io.Atom {
+	return nil
+}
+
+func (self FDummy) Tag() mp4io.Tag {
+	return self.Tag_
+}
+
+func (self FDummy) Len() int {
+	return len(self.Data)
+}
+
+func (self FDummy) Marshal(b []byte) int {
+	copy(b, self.Data)
+	return len(self.Data)
+}
+
+func (self FDummy) Unmarshal(b []byte, offset int) (n int, err error) {
+	return
+}

format/mp4f/mp4fio/atoms.go

@@ -0,0 +1,356 @@
+package mp4fio
+
+import (
+	"github.com/deepch/vdk/format/mp4/mp4io"
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+func (self MovieFrag) Tag() mp4io.Tag {
+	return mp4io.MOOF
+}
+
+type MovieFrag struct {
+	Header   *MovieFragHeader
+	Tracks   []*TrackFrag
+	Unknowns []mp4io.Atom
+	mp4io.AtomPos
+}
+
+func (self MovieFrag) Marshal(b []byte) (n int) {
+	pio.PutU32BE(b[4:], uint32(mp4io.MOOF))
+	n += self.marshal(b[8:]) + 8
+	pio.PutU32BE(b[0:], uint32(n))
+	return
+}
+func (self MovieFrag) marshal(b []byte) (n int) {
+	if self.Header != nil {
+		n += self.Header.Marshal(b[n:])
+	}
+	for _, atom := range self.Tracks {
+		n += atom.Marshal(b[n:])
+	}
+	for _, atom := range self.Unknowns {
+		n += atom.Marshal(b[n:])
+	}
+	return
+}
+func (self MovieFrag) Len() (n int) {
+	n += 8
+	if self.Header != nil {
+		n += self.Header.Len()
+	}
+	for _, atom := range self.Tracks {
+		n += atom.Len()
+	}
+	for _, atom := range self.Unknowns {
+		n += atom.Len()
+	}
+	return
+}
+func (self *MovieFrag) Unmarshal(b []byte, offset int) (n int, err error) {
+
+	return
+}
+func (self MovieFrag) Children() (r []mp4io.Atom) {
+	if self.Header != nil {
+		r = append(r, self.Header)
+	}
+	for _, atom := range self.Tracks {
+		r = append(r, atom)
+	}
+	r = append(r, self.Unknowns...)
+	return
+}
+
+func (self MovieFragHeader) Tag() mp4io.Tag {
+	return mp4io.MFHD
+}
+
+type MovieFragHeader struct {
+	Version uint8
+	Flags   uint32
+	Seqnum  uint32
+	mp4io.AtomPos
+}
+
+func (self MovieFragHeader) Marshal(b []byte) (n int) {
+	pio.PutU32BE(b[4:], uint32(mp4io.MFHD))
+	n += self.marshal(b[8:]) + 8
+	pio.PutU32BE(b[0:], uint32(n))
+	return
+}
+func (self MovieFragHeader) marshal(b []byte) (n int) {
+	pio.PutU8(b[n:], self.Version)
+	n += 1
+	pio.PutU24BE(b[n:], self.Flags)
+	n += 3
+	pio.PutU32BE(b[n:], self.Seqnum)
+	n += 4
+	return
+}
+func (self MovieFragHeader) Len() (n int) {
+	n += 8
+	n += 1
+	n += 3
+	n += 4
+	return
+}
+func (self *MovieFragHeader) Unmarshal(b []byte, offset int) (n int, err error) {
+
+	return
+}
+func (self MovieFragHeader) Children() (r []mp4io.Atom) {
+	return
+}
+
+func (self TrackFragRun) Tag() mp4io.Tag {
+	return mp4io.TRUN
+}
+
+type TrackFragRun struct {
+	Version          uint8
+	Flags            uint32
+	DataOffset       uint32
+	FirstSampleFlags uint32
+	Entries          []mp4io.TrackFragRunEntry
+	mp4io.AtomPos
+}
+
+func (self TrackFragRun) Marshal(b []byte) (n int) {
+	pio.PutU32BE(b[4:], uint32(mp4io.TRUN))
+	n += self.marshal(b[8:]) + 8
+	pio.PutU32BE(b[0:], uint32(n))
+	return
+}
+func (self TrackFragRun) marshal(b []byte) (n int) {
+	pio.PutU8(b[n:], self.Version)
+	n += 1
+	pio.PutU24BE(b[n:], self.Flags)
+	n += 3
+	pio.PutU32BE(b[n:], uint32(len(self.Entries)))
+	n += 4
+	if self.Flags&mp4io.TRUN_DATA_OFFSET != 0 {
+		{
+			pio.PutU32BE(b[n:], self.DataOffset)
+			n += 4
+		}
+	}
+	if self.Flags&mp4io.TRUN_FIRST_SAMPLE_FLAGS != 0 {
+		{
+			pio.PutU32BE(b[n:], self.FirstSampleFlags)
+			n += 4
+		}
+	}
+
+	for i, entry := range self.Entries {
+		var flags uint32
+		if i > 0 {
+			flags = self.Flags
+		} else {
+			flags = self.FirstSampleFlags
+		}
+		//if flags&TRUN_SAMPLE_DURATION != 0 {
+		pio.PutU32BE(b[n:], entry.Duration)
+		n += 4
+		//}
+		//if flags&TRUN_SAMPLE_SIZE != 0 {
+		pio.PutU32BE(b[n:], entry.Size)
+		n += 4
+		//}
+		if flags&mp4io.TRUN_SAMPLE_FLAGS != 0 {
+			pio.PutU32BE(b[n:], entry.Flags)
+			n += 4
+		}
+		//if flags&TRUN_SAMPLE_CTS != 0 {
+		pio.PutU32BE(b[n:], entry.Cts)
+		n += 4
+		//}
+	}
+	return
+}
+func (self TrackFragRun) Len() (n int) {
+	n += 8
+	n += 1
+	n += 3
+	n += 4
+	if self.Flags&mp4io.TRUN_DATA_OFFSET != 0 {
+		{
+			n += 4
+		}
+	}
+	if self.Flags&mp4io.TRUN_FIRST_SAMPLE_FLAGS != 0 {
+		{
+			n += 4
+		}
+	}
+
+	for i := range self.Entries {
+		var flags uint32
+		if i > 0 {
+			flags = self.Flags
+		} else {
+			flags = self.FirstSampleFlags
+		}
+		//if flags&TRUN_SAMPLE_DURATION != 0 {
+		n += 4
+		//}
+		//if flags&TRUN_SAMPLE_SIZE != 0 {
+		n += 4
+		//}
+		if flags&mp4io.TRUN_SAMPLE_FLAGS != 0 {
+			n += 4
+		}
+		//if flags&TRUN_SAMPLE_CTS != 0 {
+		n += 4
+		//}
+	}
+	return
+}
+func (self *TrackFragRun) Unmarshal(b []byte, offset int) (n int, err error) {
+
+	return
+}
+func (self TrackFragRun) Children() (r []mp4io.Atom) {
+	return
+}
+
+func (self TrackFrag) Tag() mp4io.Tag {
+	return mp4io.TRAF
+}
+
+type TrackFrag struct {
+	Header     *TrackFragHeader
+	DecodeTime *TrackFragDecodeTime
+	Run        *TrackFragRun
+	Unknowns   []mp4io.Atom
+	mp4io.AtomPos
+}
+
+func (self TrackFrag) Marshal(b []byte) (n int) {
+	pio.PutU32BE(b[4:], uint32(mp4io.TRAF))
+	n += self.marshal(b[8:]) + 8
+	pio.PutU32BE(b[0:], uint32(n))
+	return
+}
+func (self TrackFrag) marshal(b []byte) (n int) {
+	if self.Header != nil {
+		n += self.Header.Marshal(b[n:])
+	}
+	if self.DecodeTime != nil {
+		n += self.DecodeTime.Marshal(b[n:])
+	}
+	if self.Run != nil {
+		n += self.Run.Marshal(b[n:])
+	}
+	for _, atom := range self.Unknowns {
+		n += atom.Marshal(b[n:])
+	}
+	return
+}
+func (self TrackFrag) Len() (n int) {
+	n += 8
+	if self.Header != nil {
+		n += self.Header.Len()
+	}
+	if self.DecodeTime != nil {
+		n += self.DecodeTime.Len()
+	}
+	if self.Run != nil {
+		n += self.Run.Len()
+	}
+	for _, atom := range self.Unknowns {
+		n += atom.Len()
+	}
+	return
+}
+func (self *TrackFrag) Unmarshal(b []byte, offset int) (n int, err error) {
+
+	return
+}
+func (self TrackFrag) Children() (r []mp4io.Atom) {
+	if self.Header != nil {
+		r = append(r, self.Header)
+	}
+	if self.DecodeTime != nil {
+		r = append(r, self.DecodeTime)
+	}
+	if self.Run != nil {
+		r = append(r, self.Run)
+	}
+	r = append(r, self.Unknowns...)
+	return
+}
+
+const LenTrackFragRunEntry = 16
+
+func (self TrackFragHeader) Tag() mp4io.Tag {
+	return mp4io.TFHD
+}
+
+type TrackFragHeader struct {
+	Data []byte
+	mp4io.AtomPos
+}
+
+func (self TrackFragHeader) Marshal(b []byte) (n int) {
+	pio.PutU32BE(b[4:], uint32(mp4io.TFHD))
+	n += self.marshal(b[8:]) + 8
+	pio.PutU32BE(b[0:], uint32(n))
+	return
+}
+func (self TrackFragHeader) marshal(b []byte) (n int) {
+	copy(b, self.Data)
+	n += len(self.Data)
+	return
+}
+func (self TrackFragHeader) Len() (n int) {
+	return len(self.Data) + 8
+}
+func (self *TrackFragHeader) Unmarshal(b []byte, offset int) (n int, err error) {
+
+	return
+}
+func (self TrackFragHeader) Children() (r []mp4io.Atom) {
+	return
+}
+
+func (self TrackFragDecodeTime) Tag() mp4io.Tag {
+	return mp4io.TFDT
+}
+
+type TrackFragDecodeTime struct {
+	Version uint8
+	Flags   uint32
+	Time    uint64
+	mp4io.AtomPos
+}
+
+func (self TrackFragDecodeTime) Marshal(b []byte) (n int) {
+	pio.PutU32BE(b[4:], uint32(mp4io.TFDT))
+	n += self.marshal(b[8:]) + 8
+	pio.PutU32BE(b[0:], uint32(n))
+	return
+}
+func (self TrackFragDecodeTime) marshal(b []byte) (n int) {
+	pio.PutU8(b[n:], self.Version)
+	n += 1
+	pio.PutU24BE(b[n:], self.Flags)
+	n += 3
+	pio.PutU64BE(b[n:], self.Time)
+	n += 8
+	return
+}
+func (self TrackFragDecodeTime) Len() (n int) {
+	n += 8
+	n += 1
+	n += 3
+	n += 8
+	return
+}
+func (self *TrackFragDecodeTime) Unmarshal(b []byte, offset int) (n int, err error) {
+
+	return
+}
+func (self TrackFragDecodeTime) Children() (r []mp4io.Atom) {
+	return
+}

format/mp4f/mp4fio/gen/pattern.go

@@ -0,0 +1,437 @@
+package main
+
+func moov_Movie() {
+	atom(Header, MovieHeader)
+	atom(MovieExtend, MovieExtend)
+	atoms(Tracks, Track)
+	_unknowns()
+}
+
+func mvhd_MovieHeader() {
+	uint8(Version)
+	uint24(Flags)
+	time32(CreateTime)
+	time32(ModifyTime)
+	int32(TimeScale)
+	int32(Duration)
+	fixed32(PreferredRate)
+	fixed16(PreferredVolume)
+	_skip(10)
+	array(Matrix, int32, 9)
+	time32(PreviewTime)
+	time32(PreviewDuration)
+	time32(PosterTime)
+	time32(SelectionTime)
+	time32(SelectionDuration)
+	time32(CurrentTime)
+	int32(NextTrackId)
+}
+
+func trak_Track() {
+	atom(Header, TrackHeader)
+	atom(Media, Media)
+	_unknowns()
+}
+
+func tkhd_TrackHeader() {
+	uint8(Version)
+	uint24(Flags)
+	time32(CreateTime)
+	time32(ModifyTime)
+	int32(TrackId)
+	_skip(4)
+	int32(Duration)
+	_skip(8)
+	int16(Layer)
+	int16(AlternateGroup)
+	fixed16(Volume)
+	_skip(2)
+	array(Matrix, int32, 9)
+	fixed32(TrackWidth)
+	fixed32(TrackHeight)
+}
+
+func hdlr_HandlerRefer() {
+	uint8(Version)
+	uint24(Flags)
+	bytes(Type, 4)
+	bytes(SubType, 4)
+	bytesleft(Name)
+}
+
+func mdia_Media() {
+	atom(Header, MediaHeader)
+	atom(Handler, HandlerRefer)
+	atom(Info, MediaInfo)
+	_unknowns()
+}
+
+func mdhd_MediaHeader() {
+	uint8(Version)
+	uint24(Flags)
+	time32(CreateTime)
+	time32(ModifyTime)
+	int32(TimeScale)
+	int32(Duration)
+	int16(Language)
+	int16(Quality)
+}
+
+func minf_MediaInfo() {
+	atom(Sound, SoundMediaInfo)
+	atom(Video, VideoMediaInfo)
+	atom(Data, DataInfo)
+	atom(Sample, SampleTable)
+	_unknowns()
+}
+
+func dinf_DataInfo() {
+	atom(Refer, DataRefer)
+	_unknowns()
+}
+
+func dref_DataRefer() {
+	uint8(Version)
+	uint24(Flags)
+	int32(_childrenNR)
+	atom(Url, DataReferUrl)
+}
+
+func url__DataReferUrl() {
+	uint8(Version)
+	uint24(Flags)
+}
+
+func smhd_SoundMediaInfo() {
+	uint8(Version)
+	uint24(Flags)
+	int16(Balance)
+	_skip(2)
+}
+
+func vmhd_VideoMediaInfo() {
+	uint8(Version)
+	uint24(Flags)
+	int16(GraphicsMode)
+	array(Opcolor, int16, 3)
+}
+
+func stbl_SampleTable() {
+	atom(SampleDesc, SampleDesc)
+	atom(TimeToSample, TimeToSample)
+	atom(CompositionOffset, CompositionOffset)
+	atom(SampleToChunk, SampleToChunk)
+	atom(SyncSample, SyncSample)
+	atom(ChunkOffset, ChunkOffset)
+	atom(SampleSize, SampleSize)
+}
+
+func stsd_SampleDesc() {
+	uint8(Version)
+	_skip(3)
+	int32(_childrenNR)
+	atom(AVC1Desc, AVC1Desc)
+	atom(MP4ADesc, MP4ADesc)
+	_unknowns()
+}
+
+func mp4a_MP4ADesc() {
+	_skip(6)
+	int16(DataRefIdx)
+	int16(Version)
+	int16(RevisionLevel)
+	int32(Vendor)
+	int16(NumberOfChannels)
+	int16(SampleSize)
+	int16(CompressionId)
+	_skip(2)
+	fixed32(SampleRate)
+	atom(Conf, ElemStreamDesc)
+	_unknowns()
+}
+
+func avc1_AVC1Desc() {
+	_skip(6)
+	int16(DataRefIdx)
+	int16(Version)
+	int16(Revision)
+	int32(Vendor)
+	int32(TemporalQuality)
+	int32(SpatialQuality)
+	int16(Width)
+	int16(Height)
+	fixed32(HorizontalResolution)
+	fixed32(VorizontalResolution)
+	_skip(4)
+	int16(FrameCount)
+	bytes(CompressorName, 32)
+	int16(Depth)
+	int16(ColorTableId)
+	atom(Conf, AVC1Conf)
+	_unknowns()
+}
+
+func avcC_AVC1Conf() {
+	bytesleft(Data)
+}
+
+func stts_TimeToSample() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, TimeToSampleEntry)
+}
+
+func TimeToSampleEntry() {
+	uint32(Count)
+	uint32(Duration)
+}
+
+func stsc_SampleToChunk() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, SampleToChunkEntry)
+}
+
+func SampleToChunkEntry() {
+	uint32(FirstChunk)
+	uint32(SamplesPerChunk)
+	uint32(SampleDescId)
+}
+
+func ctts_CompositionOffset() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, CompositionOffsetEntry)
+}
+
+func CompositionOffsetEntry() {
+	uint32(Count)
+	uint32(Offset)
+}
+
+func stss_SyncSample() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, uint32)
+}
+
+func stco_ChunkOffset() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+	slice(Entries, uint32)
+}
+
+func moof_MovieFrag() {
+	atom(Header, MovieFragHeader)
+	atoms(Tracks, TrackFrag)
+	_unknowns()
+}
+
+func mfhd_MovieFragHeader() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(Seqnum)
+}
+
+func traf_TrackFrag() {
+	atom(Header, TrackFragHeader)
+	atom(DecodeTime, TrackFragDecodeTime)
+	atom(Run, TrackFragRun)
+	_unknowns()
+}
+
+func mvex_MovieExtend() {
+	atoms(Tracks, TrackExtend)
+	_unknowns()
+}
+
+func trex_TrackExtend() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(TrackId)
+	uint32(DefaultSampleDescIdx)
+	uint32(DefaultSampleDuration)
+	uint32(DefaultSampleSize)
+	uint32(DefaultSampleFlags)
+}
+
+func stsz_SampleSize() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(SampleSize)
+	_code(func() {
+		if self.SampleSize != 0 {
+			return
+		}
+	})
+	uint32(_len_Entries)
+	slice(Entries, uint32)
+}
+
+func trun_TrackFragRun() {
+	uint8(Version)
+	uint24(Flags)
+	uint32(_len_Entries)
+
+	uint32(DataOffset, _code(func() {
+		if self.Flags&TRUN_DATA_OFFSET != 0 {
+			doit()
+		}
+	}))
+
+	uint32(FirstSampleFlags, _code(func() {
+		if self.Flags&TRUN_FIRST_SAMPLE_FLAGS != 0 {
+			doit()
+		}
+	}))
+
+	slice(Entries, TrackFragRunEntry, _code(func() {
+		for i, entry := range self.Entries {
+			var flags uint32
+			if i > 0 {
+				flags = self.Flags
+			} else {
+				flags = self.FirstSampleFlags
+			}
+			if flags&TRUN_SAMPLE_DURATION != 0 {
+				pio.PutU32BE(b[n:], entry.Duration)
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_SIZE != 0 {
+				pio.PutU32BE(b[n:], entry.Size)
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_FLAGS != 0 {
+				pio.PutU32BE(b[n:], entry.Flags)
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_CTS != 0 {
+				pio.PutU32BE(b[n:], entry.Cts)
+				n += 4
+			}
+		}
+	}, func() {
+		for i := range self.Entries {
+			var flags uint32
+			if i > 0 {
+				flags = self.Flags
+			} else {
+				flags = self.FirstSampleFlags
+			}
+			if flags&TRUN_SAMPLE_DURATION != 0 {
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_SIZE != 0 {
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_FLAGS != 0 {
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_CTS != 0 {
+				n += 4
+			}
+		}
+	}, func() {
+		for i := 0; i < int(_len_Entries); i++ {
+			var flags uint32
+			if i > 0 {
+				flags = self.Flags
+			} else {
+				flags = self.FirstSampleFlags
+			}
+			entry := &self.Entries[i]
+			if flags&TRUN_SAMPLE_DURATION != 0 {
+				entry.Duration = pio.U32BE(b[n:])
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_SIZE != 0 {
+				entry.Size = pio.U32BE(b[n:])
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_FLAGS != 0 {
+				entry.Flags = pio.U32BE(b[n:])
+				n += 4
+			}
+			if flags&TRUN_SAMPLE_CTS != 0 {
+				entry.Cts = pio.U32BE(b[n:])
+				n += 4
+			}
+		}
+	}))
+}
+
+func TrackFragRunEntry() {
+	uint32(Duration)
+	uint32(Size)
+	uint32(Flags)
+	uint32(Cts)
+}
+
+func tfhd_TrackFragHeader() {
+	uint8(Version)
+	uint24(Flags)
+
+	uint64(BaseDataOffset, _code(func() {
+		if self.Flags&TFHD_BASE_DATA_OFFSET != 0 {
+			doit()
+		}
+	}))
+
+	uint32(StsdId, _code(func() {
+		if self.Flags&TFHD_STSD_ID != 0 {
+			doit()
+		}
+	}))
+
+	uint32(DefaultDuration, _code(func() {
+		if self.Flags&TFHD_DEFAULT_DURATION != 0 {
+			doit()
+		}
+	}))
+
+	uint32(DefaultSize, _code(func() {
+		if self.Flags&TFHD_DEFAULT_SIZE != 0 {
+			doit()
+		}
+	}))
+
+	uint32(DefaultFlags, _code(func() {
+		if self.Flags&TFHD_DEFAULT_FLAGS != 0 {
+			doit()
+		}
+	}))
+}
+
+func tfdt_TrackFragDecodeTime() {
+	uint8(Version)
+	uint24(Flags)
+	time64(Time, _code(func() {
+		if self.Version != 0 {
+			PutTime64(b[n:], self.Time)
+			n += 8
+		} else {
+			PutTime32(b[n:], self.Time)
+			n += 4
+		}
+	}, func() {
+		if self.Version != 0 {
+			n += 8
+		} else {
+			n += 4
+		}
+	}, func() {
+		if self.Version != 0 {
+			self.Time = GetTime64(b[n:])
+			n += 8
+		} else {
+			self.Time = GetTime32(b[n:])
+			n += 4
+		}
+	}))
+}
+

format/mp4f/mp4fio/mp4io.go

@@ -0,0 +1,113 @@
+package mp4fio
+
+import (
+	"github.com/deepch/vdk/format/mp4/mp4io"
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+type ElemStreamDesc struct {
+	DecConfig []byte
+	TrackId   uint16
+	mp4io.AtomPos
+}
+
+func (self ElemStreamDesc) Children() []mp4io.Atom {
+	return nil
+}
+
+func (self ElemStreamDesc) fillLength(b []byte, length int) (n int) {
+	b[n] = uint8(length & 0x7f)
+	n++
+	return
+}
+
+func (self ElemStreamDesc) lenDescHdr() (n int) {
+	return 2
+}
+
+func (self ElemStreamDesc) fillDescHdr(b []byte, tag uint8, datalen int) (n int) {
+	b[n] = tag
+	n++
+	n += self.fillLength(b[n:], datalen)
+	return
+}
+
+func (self ElemStreamDesc) lenESDescHdr() (n int) {
+	return self.lenDescHdr() + 3
+}
+
+func (self ElemStreamDesc) fillESDescHdr(b []byte, datalen int) (n int) {
+	n += self.fillDescHdr(b[n:], mp4io.MP4ESDescrTag, datalen)
+	pio.PutU16BE(b[n:], self.TrackId)
+	n += 2
+	b[n] = 0 // flags
+	n++
+	return
+}
+
+func (self ElemStreamDesc) lenDecConfigDescHdr() (n int) {
+	return self.lenDescHdr() + 2 + 3 + 4 + 4 + self.lenDescHdr()
+}
+
+func (self ElemStreamDesc) fillDecConfigDescHdr(b []byte, datalen int) (n int) {
+	n += self.fillDescHdr(b[n:], mp4io.MP4DecConfigDescrTag, datalen)
+	b[n] = 0x40 // objectid
+	n++
+	b[n] = 0x15 // streamtype
+	n++
+	// buffer size db
+	pio.PutU24BE(b[n:], 0)
+	n += 3
+	// max bitrage
+	pio.PutU32BE(b[n:], uint32(200000))
+	n += 4
+	// avg bitrage
+	pio.PutU32BE(b[n:], uint32(0))
+	n += 4
+	n += self.fillDescHdr(b[n:], mp4io.MP4DecSpecificDescrTag, datalen-n)
+	return
+}
+
+func (self ElemStreamDesc) Len() (n int) {
+	// len + tag
+	return 8 +
+		// ver + flags
+		4 +
+		self.lenESDescHdr() +
+		self.lenDecConfigDescHdr() +
+		len(self.DecConfig) +
+		self.lenDescHdr() + 1
+}
+
+// Version(4)
+// ESDesc(
+//   MP4ESDescrTag
+//   ESID(2)
+//   ESFlags(1)
+//   DecConfigDesc(
+//     MP4DecConfigDescrTag
+//     objectId streamType bufSize avgBitrate
+//     DecSpecificDesc(
+//       MP4DecSpecificDescrTag
+//       decConfig
+//     )
+//   )
+//   ?Desc(lenDescHdr+1)
+// )
+
+func (self ElemStreamDesc) Marshal(b []byte) (n int) {
+	pio.PutU32BE(b[4:], uint32(mp4io.ESDS))
+	n += 8
+	pio.PutU32BE(b[n:], 0) // Version
+	n += 4
+	datalen := self.Len()
+	n += self.fillESDescHdr(b[n:], datalen-n-self.lenESDescHdr()+3)
+	n += self.fillDecConfigDescHdr(b[n:], datalen-n-self.lenDescHdr()-3)
+	copy(b[n:], self.DecConfig)
+	n += len(self.DecConfig)
+	n += self.fillDescHdr(b[n:], 0x06, datalen-n-self.lenDescHdr())
+	b[n] = 0x02
+	n++
+	pio.PutU32BE(b[0:], uint32(n))
+	return
+}

format/mp4f/muxer.go

@@ -0,0 +1,402 @@
+package mp4f
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/codec/aacparser"
+	"github.com/deepch/vdk/codec/h264parser"
+	"github.com/deepch/vdk/format/mp4/mp4io"
+	"github.com/deepch/vdk/format/mp4f/mp4fio"
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+type Muxer struct {
+	maxFrames     int
+	bufw          *bufio.Writer
+	wpos          int64
+	fragmentIndex int
+	streams       []*Stream
+	path          string
+}
+
+func NewMuxer(w *os.File) *Muxer {
+	return &Muxer{}
+}
+func (self *Muxer) SetPath(path string) {
+	self.path = path
+}
+func (self *Muxer) SetMaxFrames(count int) {
+	self.maxFrames = count
+}
+func (self *Muxer) newStream(codec av.CodecData) (err error) {
+	switch codec.Type() {
+	case av.H264, av.AAC:
+	default:
+		err = fmt.Errorf("fmp4: codec type=%v is not supported", codec.Type())
+		return
+	}
+	stream := &Stream{CodecData: codec}
+
+	stream.sample = &mp4io.SampleTable{
+		SampleDesc:    &mp4io.SampleDesc{},
+		TimeToSample:  &mp4io.TimeToSample{},
+		SampleToChunk: &mp4io.SampleToChunk{},
+		SampleSize:    &mp4io.SampleSize{},
+		ChunkOffset:   &mp4io.ChunkOffset{},
+	}
+
+	stream.trackAtom = &mp4io.Track{
+		Header: &mp4io.TrackHeader{
+			TrackId:  int32(len(self.streams) + 1),
+			Flags:    0x0007,
+			Duration: 0,
+			Matrix:   [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
+		},
+		Media: &mp4io.Media{
+			Header: &mp4io.MediaHeader{
+				TimeScale: 1000,
+				Duration:  0,
+				Language:  21956,
+			},
+			Info: &mp4io.MediaInfo{
+				Sample: stream.sample,
+				Data: &mp4io.DataInfo{
+					Refer: &mp4io.DataRefer{
+						Url: &mp4io.DataReferUrl{
+							Flags: 0x000001,
+						},
+					},
+				},
+			},
+		},
+	}
+	switch codec.Type() {
+	case av.H264:
+		stream.sample.SyncSample = &mp4io.SyncSample{}
+		stream.timeScale = 90000
+	case av.AAC:
+		stream.timeScale = 8000
+	}
+
+	stream.muxer = self
+	self.streams = append(self.streams, stream)
+
+	return
+}
+
+func (self *Stream) buildEsds(conf []byte) *FDummy {
+	esds := &mp4fio.ElemStreamDesc{DecConfig: conf}
+
+	b := make([]byte, esds.Len())
+	esds.Marshal(b)
+
+	esdsDummy := FDummy{
+		Data: b,
+		Tag_: mp4io.Tag(uint32(mp4io.ESDS)),
+	}
+	return &esdsDummy
+}
+
+func (self *Stream) buildHdlr() *FDummy {
+	hdlr := FDummy{
+		Data: []byte{
+			0x00, 0x00, 0x00, 0x35, 0x68, 0x64, 0x6C, 0x72,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x69,
+			0x64, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x47, 0x6F, 0x6F, 0x64, 0x67, 0x61,
+			0x6D, 0x65, 0x20, 0x47, 0x4F, 0x20, 0x53, 0x65, 0x72, 0x76,
+			0x65, 0x72, 0x00, 0x00, 0x00},
+
+		Tag_: mp4io.Tag(uint32(mp4io.HDLR)),
+	}
+	return &hdlr
+}
+
+func (self *Stream) buildAudioHdlr() *FDummy {
+	hdlr := FDummy{
+		Data: []byte{
+			0x00, 0x00, 0x00, 0x35, 0x68, 0x64, 0x6C, 0x72,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x6F,
+			0x75, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x42, 0x65, 0x6E, 0x74, 0x6F, 0x34,
+			0x20, 0x53, 0x6F, 0x75, 0x6E, 0x64, 0x20, 0x48, 0x61, 0x6E,
+			0x64, 0x6C, 0x65, 0x72, 0x00},
+
+		Tag_: mp4io.Tag(uint32(mp4io.HDLR)),
+	}
+	return &hdlr
+}
+
+func (self *Stream) buildEdts() *FDummy {
+	edts := FDummy{
+		Data: []byte{
+			0x00, 0x00, 0x00, 0x30, 0x65, 0x64, 0x74, 0x73,
+			0x00, 0x00, 0x00, 0x28, 0x65, 0x6C, 0x73, 0x74, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x21,
+			0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
+			0x9B, 0x24, 0x00, 0x00, 0x02, 0x10, 0x00, 0x01, 0x00, 0x00,
+		},
+		Tag_: mp4io.Tag(0x65647473),
+	}
+	return &edts
+}
+
+func (self *Stream) fillTrackAtom() (err error) {
+	self.trackAtom.Media.Header.TimeScale = int32(self.timeScale)
+	self.trackAtom.Media.Header.Duration = int32(self.duration)
+
+	if self.Type() == av.H264 {
+		codec := self.CodecData.(h264parser.CodecData)
+		width, height := codec.Width(), codec.Height()
+		self.sample.SampleDesc.AVC1Desc = &mp4io.AVC1Desc{
+			DataRefIdx:           1,
+			HorizontalResolution: 72,
+			VorizontalResolution: 72,
+			Width:                int16(width),
+			Height:               int16(height),
+			FrameCount:           1,
+			Depth:                24,
+			ColorTableId:         -1,
+			Conf:                 &mp4io.AVC1Conf{Data: codec.AVCDecoderConfRecordBytes()},
+		}
+		self.trackAtom.Header.TrackWidth = float64(width)
+		self.trackAtom.Header.TrackHeight = float64(height)
+
+		self.trackAtom.Media.Handler = &mp4io.HandlerRefer{
+			SubType: [4]byte{'v', 'i', 'd', 'e'},
+			Name:    []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'G', 'G', 0, 0, 0},
+		}
+		self.trackAtom.Media.Info.Video = &mp4io.VideoMediaInfo{
+			Flags: 0x000001,
+		}
+		self.codecString = fmt.Sprintf("avc1.%02X%02X%02X", codec.RecordInfo.AVCProfileIndication, codec.RecordInfo.ProfileCompatibility, codec.RecordInfo.AVCLevelIndication)
+	} else if self.Type() == av.AAC {
+		codec := self.CodecData.(aacparser.CodecData)
+		self.sample.SampleDesc.MP4ADesc = &mp4io.MP4ADesc{
+			DataRefIdx:       1,
+			NumberOfChannels: int16(codec.ChannelLayout().Count()),
+			SampleSize:       16,
+			SampleRate:       float64(codec.SampleRate()),
+			Unknowns:         []mp4io.Atom{self.buildEsds(codec.MPEG4AudioConfigBytes())},
+		}
+		self.trackAtom.Header.Volume = 1
+		self.trackAtom.Header.AlternateGroup = 1
+		self.trackAtom.Header.Duration = 0
+
+		self.trackAtom.Media.Handler = &mp4io.HandlerRefer{
+			SubType: [4]byte{'s', 'o', 'u', 'n'},
+			Name:    []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'G', 'G', 0, 0, 0},
+		}
+
+		self.trackAtom.Media.Info.Sound = &mp4io.SoundMediaInfo{}
+		self.codecString = "mp4a.40.2"
+
+	} else {
+		err = fmt.Errorf("fmp4: codec type=%d invalid", self.Type())
+	}
+
+	return
+}
+
+func (self *Muxer) WriteTrailer() (err error) {
+	return
+}
+
+func (element *Muxer) WriteHeader(streams []av.CodecData) (err error) {
+	element.streams = []*Stream{}
+	for _, stream := range streams {
+		if err = element.newStream(stream); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (element *Muxer) GetInit(streams []av.CodecData) (string, []byte) {
+	moov := &mp4io.Movie{
+		Header: &mp4io.MovieHeader{
+			PreferredRate:     1,
+			PreferredVolume:   1,
+			Matrix:            [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
+			NextTrackId:       3,
+			Duration:          0,
+			TimeScale:         1000,
+			CreateTime:        time0(),
+			ModifyTime:        time0(),
+			PreviewTime:       time0(),
+			PreviewDuration:   time0(),
+			PosterTime:        time0(),
+			SelectionTime:     time0(),
+			SelectionDuration: time0(),
+			CurrentTime:       time0(),
+		},
+		Unknowns: []mp4io.Atom{element.buildMvex()},
+	}
+	var meta string
+	for _, stream := range element.streams {
+		if err := stream.fillTrackAtom(); err != nil {
+			return meta, []byte{}
+		}
+		moov.Tracks = append(moov.Tracks, stream.trackAtom)
+		meta += stream.codecString + ","
+	}
+	meta = meta[:len(meta)-1]
+	ftypeData := []byte{0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x36, 0x00, 0x00, 0x00, 0x01, 0x69, 0x73, 0x6f, 0x36, 0x64, 0x61, 0x73, 0x68}
+	file := make([]byte, moov.Len()+len(ftypeData))
+	copy(file, ftypeData)
+	moov.Marshal(file[len(ftypeData):])
+	return meta, file
+}
+
+func (element *Muxer) WritePacket(pkt av.Packet, GOP bool) (bool, []byte, error) {
+	stream := element.streams[pkt.Idx]
+	if GOP {
+		ts := time.Duration(0)
+		if stream.lastpkt != nil {
+			ts = pkt.Time - stream.lastpkt.Time
+		}
+		got, buf, err := stream.writePacketV3(pkt, ts, 5)
+		stream.lastpkt = &pkt
+		if err != nil {
+			return false, []byte{}, err
+		}
+		return got, buf, err
+	}
+	ts := time.Duration(0)
+	if stream.lastpkt != nil {
+		ts = pkt.Time - stream.lastpkt.Time
+	}
+	got, buf, err := stream.writePacketV2(pkt, ts, 5)
+	stream.lastpkt = &pkt
+	if err != nil {
+		return false, []byte{}, err
+	}
+	return got, buf, err
+}
+
+func (element *Stream) writePacketV3(pkt av.Packet, rawdur time.Duration, maxFrames int) (bool, []byte, error) {
+	trackID := pkt.Idx + 1
+	var out []byte
+	var got bool
+	if element.sampleIndex > maxFrames && pkt.IsKeyFrame {
+		element.moof.Tracks[0].Run.DataOffset = uint32(element.moof.Len() + 8)
+		out = make([]byte, element.moof.Len()+len(element.buffer))
+		element.moof.Marshal(out)
+		pio.PutU32BE(element.buffer, uint32(len(element.buffer)))
+		copy(out[element.moof.Len():], element.buffer)
+		element.sampleIndex = 0
+		element.muxer.fragmentIndex++
+		got = true
+	}
+	if element.sampleIndex == 0 {
+		element.moof.Header = &mp4fio.MovieFragHeader{Seqnum: uint32(element.muxer.fragmentIndex + 1)}
+		element.moof.Tracks = []*mp4fio.TrackFrag{
+			&mp4fio.TrackFrag{
+				Header: &mp4fio.TrackFragHeader{
+					Data: []byte{0x00, 0x02, 0x00, 0x20, 0x00, 0x00, 0x00, uint8(trackID), 0x01, 0x01, 0x00, 0x00},
+				},
+				DecodeTime: &mp4fio.TrackFragDecodeTime{
+					Version: 1,
+					Flags:   0,
+					Time:    uint64(element.dts),
+				},
+				Run: &mp4fio.TrackFragRun{
+					Flags:            0x000b05,
+					FirstSampleFlags: 0x02000000,
+					DataOffset:       0,
+					Entries:          []mp4io.TrackFragRunEntry{},
+				},
+			},
+		}
+		element.buffer = []byte{0x00, 0x00, 0x00, 0x00, 0x6d, 0x64, 0x61, 0x74}
+	}
+	runEnrty := mp4io.TrackFragRunEntry{
+		Duration: uint32(element.timeToTs(rawdur)),
+		Size:     uint32(len(pkt.Data)),
+		Cts:      uint32(element.timeToTs(pkt.CompositionTime)),
+	}
+	element.moof.Tracks[0].Run.Entries = append(element.moof.Tracks[0].Run.Entries, runEnrty)
+	element.buffer = append(element.buffer, pkt.Data...)
+	element.sampleIndex++
+	element.dts += element.timeToTs(rawdur)
+	return got, out, nil
+}
+
+func (element *Stream) writePacketV2(pkt av.Packet, rawdur time.Duration, maxFrames int) (bool, []byte, error) {
+	trackID := pkt.Idx + 1
+	if element.sampleIndex == 0 {
+		element.moof.Header = &mp4fio.MovieFragHeader{Seqnum: uint32(element.muxer.fragmentIndex + 1)}
+		element.moof.Tracks = []*mp4fio.TrackFrag{
+			&mp4fio.TrackFrag{
+				Header: &mp4fio.TrackFragHeader{
+					Data: []byte{0x00, 0x02, 0x00, 0x20, 0x00, 0x00, 0x00, uint8(trackID), 0x01, 0x01, 0x00, 0x00},
+				},
+				DecodeTime: &mp4fio.TrackFragDecodeTime{
+					Version: 1,
+					Flags:   0,
+					Time:    uint64(element.dts),
+				},
+				Run: &mp4fio.TrackFragRun{
+					Flags:            0x000b05,
+					FirstSampleFlags: 0x02000000,
+					DataOffset:       0,
+					Entries:          []mp4io.TrackFragRunEntry{},
+				},
+			},
+		}
+		element.buffer = []byte{0x00, 0x00, 0x00, 0x00, 0x6d, 0x64, 0x61, 0x74}
+	}
+	runEnrty := mp4io.TrackFragRunEntry{
+		Duration: uint32(element.timeToTs(rawdur)),
+		Size:     uint32(len(pkt.Data)),
+		Cts:      uint32(element.timeToTs(pkt.CompositionTime)),
+	}
+	element.moof.Tracks[0].Run.Entries = append(element.moof.Tracks[0].Run.Entries, runEnrty)
+	element.buffer = append(element.buffer, pkt.Data...)
+	element.sampleIndex++
+	element.dts += element.timeToTs(rawdur)
+	if element.sampleIndex > maxFrames { // Количество фреймов в пакете
+		element.moof.Tracks[0].Run.DataOffset = uint32(element.moof.Len() + 8)
+		file := make([]byte, element.moof.Len()+len(element.buffer))
+		element.moof.Marshal(file)
+		pio.PutU32BE(element.buffer, uint32(len(element.buffer)))
+		copy(file[element.moof.Len():], element.buffer)
+		element.sampleIndex = 0
+		element.muxer.fragmentIndex++
+		return true, file, nil
+	}
+	return false, []byte{}, nil
+}
+
+func (self *Muxer) buildMvex() *FDummy {
+	mvex := &FDummy{
+		Data: []byte{
+			0x00, 0x00, 0x00, 0x38, 0x6D, 0x76, 0x65, 0x78,
+			0x00, 0x00, 0x00, 0x10, 0x6D, 0x65, 0x68, 0x64, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+		Tag_: mp4io.Tag(0x6D766578),
+	}
+	for i := 1; i <= len(self.streams); i++ {
+		trex := self.buildTrex(i)
+		mvex.Data = append(mvex.Data, trex...)
+	}
+
+	pio.PutU32BE(mvex.Data, uint32(len(mvex.Data)))
+	return mvex
+}
+
+func (self *Muxer) buildTrex(trackId int) []byte {
+	return []byte{
+		0x00, 0x00, 0x00, 0x20, 0x74, 0x72, 0x65, 0x78,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, uint8(trackId), 0x00, 0x00,
+		0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00}
+}
+
+func time0() time.Time {
+	return time.Date(1904, time.January, 1, 0, 0, 0, 0, time.UTC)
+}

format/mp4f/stream.go

@@ -0,0 +1,54 @@
+package mp4f
+
+import (
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/format/mp4"
+	"github.com/deepch/vdk/format/mp4/mp4io"
+	"github.com/deepch/vdk/format/mp4f/mp4fio"
+)
+
+type Stream struct {
+	av.CodecData
+	codecString            string
+	trackAtom              *mp4io.Track
+	idx                    int
+	lastpkt                *av.Packet
+	timeScale              int64
+	duration               int64
+	muxer                  *Muxer
+	demuxer                *mp4.Demuxer
+	sample                 *mp4io.SampleTable
+	sampleIndex            int
+	sampleOffsetInChunk    int64
+	syncSampleIndex        int
+	dts                    int64
+	sttsEntryIndex         int
+	sampleIndexInSttsEntry int
+	cttsEntryIndex         int
+	sampleIndexInCttsEntry int
+	chunkGroupIndex        int
+	chunkIndex             int
+	sampleIndexInChunk     int
+	sttsEntry              *mp4io.TimeToSampleEntry
+	cttsEntry              *mp4io.CompositionOffsetEntry
+	moof                   mp4fio.MovieFrag
+	buffer                 []byte
+}
+
+func timeToTs(tm time.Duration, timeScale int64) int64 {
+	return int64(tm * time.Duration(timeScale) / time.Second)
+}
+
+func tsToTime(ts int64, timeScale int64) time.Duration {
+	return time.Duration(ts) * time.Second / time.Duration(timeScale)
+}
+
+func (obj *Stream) timeToTs(tm time.Duration) int64 {
+	return int64(tm * time.Duration(obj.timeScale) / time.Second)
+}
+
+func (obj *Stream) tsToTime(ts int64) time.Duration {
+	return time.Duration(ts) * time.Second / time.Duration(obj.timeScale)
+}

format/rtsp/conn.go

@@ -0,0 +1,25 @@
+package rtsp
+
+import (
+	"net"
+	"time"
+)
+
+type connWithTimeout struct {
+	Timeout time.Duration
+	net.Conn
+}
+
+func (self connWithTimeout) Read(p []byte) (n int, err error) {
+	if self.Timeout > 0 {
+		self.Conn.SetReadDeadline(time.Now().Add(self.Timeout))
+	}
+	return self.Conn.Read(p)
+}
+
+func (self connWithTimeout) Write(p []byte) (n int, err error) {
+	if self.Timeout > 0 {
+		self.Conn.SetWriteDeadline(time.Now().Add(self.Timeout))
+	}
+	return self.Conn.Write(p)
+}

format/rtsp/sdp/parser.go

@@ -0,0 +1,117 @@
+package sdp
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/deepch/vdk/av"
+)
+
+type Session struct {
+	Uri string
+}
+
+type Media struct {
+	AVType             string
+	Type               av.CodecType
+	TimeScale          int
+	Control            string
+	Rtpmap             int
+	Config             []byte
+	SpropParameterSets [][]byte
+	PayloadType        int
+	SizeLength         int
+	IndexLength        int
+}
+
+func Parse(content string) (sess Session, medias []Media) {
+	var media *Media
+
+	for _, line := range strings.Split(content, "\n") {
+		line = strings.TrimSpace(line)
+		typeval := strings.SplitN(line, "=", 2)
+		if len(typeval) == 2 {
+			fields := strings.SplitN(typeval[1], " ", 2)
+
+			switch typeval[0] {
+			case "m":
+				if len(fields) > 0 {
+					switch fields[0] {
+					case "audio", "video":
+						medias = append(medias, Media{AVType: fields[0]})
+						media = &medias[len(medias)-1]
+						mfields := strings.Split(fields[1], " ")
+						if len(mfields) >= 3 {
+							media.PayloadType, _ = strconv.Atoi(mfields[2])
+						}
+					}
+				}
+
+			case "u":
+				sess.Uri = typeval[1]
+
+			case "a":
+				if media != nil {
+					for _, field := range fields {
+						keyval := strings.SplitN(field, ":", 2)
+						if len(keyval) >= 2 {
+							key := keyval[0]
+							val := keyval[1]
+							switch key {
+							case "control":
+								media.Control = val
+							case "rtpmap":
+								media.Rtpmap, _ = strconv.Atoi(val)
+							}
+						}
+						keyval = strings.Split(field, "/")
+						if len(keyval) >= 2 {
+							key := keyval[0]
+							switch strings.ToUpper(key) {
+							case "MPEG4-GENERIC":
+								media.Type = av.AAC
+							case "H264":
+								media.Type = av.H264
+							}
+							if i, err := strconv.Atoi(keyval[1]); err == nil {
+								media.TimeScale = i
+							}
+							if false {
+								fmt.Println("sdp:", keyval[1], media.TimeScale)
+							}
+						}
+						keyval = strings.Split(field, ";")
+						if len(keyval) > 1 {
+							for _, field := range keyval {
+								keyval := strings.SplitN(field, "=", 2)
+								if len(keyval) == 2 {
+									key := strings.TrimSpace(keyval[0])
+									val := keyval[1]
+									switch key {
+									case "config":
+										media.Config, _ = hex.DecodeString(val)
+									case "sizelength":
+										media.SizeLength, _ = strconv.Atoi(val)
+									case "indexlength":
+										media.IndexLength, _ = strconv.Atoi(val)
+									case "sprop-parameter-sets":
+										fields := strings.Split(val, ",")
+										for _, field := range fields {
+											val, _ := base64.StdEncoding.DecodeString(field)
+											media.SpropParameterSets = append(media.SpropParameterSets, val)
+										}
+									}
+								}
+							}
+						}
+					}
+				}
+
+			}
+		}
+	}
+	return
+}

format/rtsp/sdp/parser_test.go

@@ -0,0 +1,44 @@
+package sdp
+
+import (
+	"testing"
+)
+
+func TestParse(t *testing.T) {
+	infos := Decode(`
+v=0
+o=- 1459325504777324 1 IN IP4 192.168.0.123
+s=RTSP/RTP stream from Network Video Server
+i=mpeg4cif
+t=0 0
+a=tool:LIVE555 Streaming Media v2009.09.28
+a=type:broadcast
+a=control:*
+a=range:npt=0-
+a=x-qt-text-nam:RTSP/RTP stream from Network Video Server
+a=x-qt-text-inf:mpeg4cif
+m=video 0 RTP/AVP 96
+c=IN IP4 0.0.0.0
+b=AS:300
+a=rtpmap:96 H264/90000
+a=fmtp:96 profile-level-id=420029; packetization-mode=1; sprop-parameter-sets=Z00AHpWoKA9k,aO48gA==
+a=x-dimensions: 720, 480
+a=x-framerate: 15
+a=control:track1
+m=audio 0 RTP/AVP 96
+c=IN IP4 0.0.0.0
+b=AS:256
+a=rtpmap:96 MPEG4-GENERIC/16000/2
+a=fmtp:96 streamtype=5;profile-level-id=1;mode=AAC-hbr;sizelength=13;indexlength=3;indexdeltalength=3;config=1408
+a=control:track2
+m=audio 0 RTP/AVP 0
+c=IN IP4 0.0.0.0
+b=AS:50
+a=recvonly
+a=control:rtsp://109.195.127.207:554/mpeg4cif/trackID=2
+a=rtpmap:0 PCMU/8000
+a=Media_header:MEDIAINFO=494D4B48010100000400010010710110401F000000FA000000000000000000000000000000000000;
+a=appversion:1.0
+`)
+	t.Logf("%v", infos)
+}

format/rtsp/stream.go

@@ -0,0 +1,29 @@
+package rtsp
+
+import (
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/format/rtsp/sdp"
+)
+
+type Stream struct {
+	av.CodecData
+	Sdp    sdp.Media
+	client *Client
+
+	// h264
+	fuStarted  bool
+	fuBuffer   []byte
+	sps        []byte
+	pps        []byte
+	spsChanged bool
+	ppsChanged bool
+
+	gotpkt         bool
+	pkt            av.Packet
+	timestamp      uint32
+	firsttimestamp uint32
+
+	lasttime time.Duration
+}

format/ts/demuxer.go

@@ -0,0 +1,291 @@
+package ts
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/codec/aacparser"
+	"github.com/deepch/vdk/codec/h264parser"
+	"github.com/deepch/vdk/format/ts/tsio"
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+type Demuxer struct {
+	r *bufio.Reader
+
+	pkts []av.Packet
+
+	pat     *tsio.PAT
+	pmt     *tsio.PMT
+	streams []*Stream
+	tshdr   []byte
+
+	stage int
+}
+
+func NewDemuxer(r io.Reader) *Demuxer {
+	return &Demuxer{
+		tshdr: make([]byte, 188),
+		r:     bufio.NewReaderSize(r, pio.RecommendBufioSize),
+	}
+}
+
+func (self *Demuxer) Streams() (streams []av.CodecData, err error) {
+	if err = self.probe(); err != nil {
+		return
+	}
+	for _, stream := range self.streams {
+		streams = append(streams, stream.CodecData)
+	}
+	return
+}
+
+func (self *Demuxer) probe() (err error) {
+	if self.stage == 0 {
+		for {
+			if self.pmt != nil {
+				n := 0
+				for _, stream := range self.streams {
+					if stream.CodecData != nil {
+						n++
+					}
+				}
+				if n == len(self.streams) {
+					break
+				}
+			}
+			if err = self.poll(); err != nil {
+				return
+			}
+		}
+		self.stage++
+	}
+	return
+}
+
+func (self *Demuxer) ReadPacket() (pkt av.Packet, err error) {
+	if err = self.probe(); err != nil {
+		return
+	}
+
+	for len(self.pkts) == 0 {
+		if err = self.poll(); err != nil {
+			return
+		}
+	}
+
+	pkt = self.pkts[0]
+	self.pkts = self.pkts[1:]
+	return
+}
+
+func (self *Demuxer) poll() (err error) {
+	if err = self.readTSPacket(); err == io.EOF {
+		var n int
+		if n, err = self.payloadEnd(); err != nil {
+			return
+		}
+		if n == 0 {
+			err = io.EOF
+		}
+	}
+	return
+}
+
+func (self *Demuxer) initPMT(payload []byte) (err error) {
+	var psihdrlen int
+	var datalen int
+	if _, _, psihdrlen, datalen, err = tsio.ParsePSI(payload); err != nil {
+		return
+	}
+	self.pmt = &tsio.PMT{}
+	if _, err = self.pmt.Unmarshal(payload[psihdrlen : psihdrlen+datalen]); err != nil {
+		return
+	}
+
+	self.streams = []*Stream{}
+	for i, info := range self.pmt.ElementaryStreamInfos {
+		stream := &Stream{}
+		stream.idx = i
+		stream.demuxer = self
+		stream.pid = info.ElementaryPID
+		stream.streamType = info.StreamType
+		switch info.StreamType {
+		case tsio.ElementaryStreamTypeH264:
+			self.streams = append(self.streams, stream)
+		case tsio.ElementaryStreamTypeAdtsAAC:
+			self.streams = append(self.streams, stream)
+		}
+	}
+	return
+}
+
+func (self *Demuxer) payloadEnd() (n int, err error) {
+	for _, stream := range self.streams {
+		var i int
+		if i, err = stream.payloadEnd(); err != nil {
+			return
+		}
+		n += i
+	}
+	return
+}
+
+func (self *Demuxer) readTSPacket() (err error) {
+	var hdrlen int
+	var pid uint16
+	var start bool
+	var iskeyframe bool
+
+	if _, err = io.ReadFull(self.r, self.tshdr); err != nil {
+		return
+	}
+
+	if pid, start, iskeyframe, hdrlen, err = tsio.ParseTSHeader(self.tshdr); err != nil {
+		return
+	}
+	payload := self.tshdr[hdrlen:]
+
+	if self.pat == nil {
+		if pid == 0 {
+			var psihdrlen int
+			var datalen int
+			if _, _, psihdrlen, datalen, err = tsio.ParsePSI(payload); err != nil {
+				return
+			}
+			self.pat = &tsio.PAT{}
+			if _, err = self.pat.Unmarshal(payload[psihdrlen : psihdrlen+datalen]); err != nil {
+				return
+			}
+		}
+	} else if self.pmt == nil {
+		for _, entry := range self.pat.Entries {
+			if entry.ProgramMapPID == pid {
+				if err = self.initPMT(payload); err != nil {
+					return
+				}
+				break
+			}
+		}
+	} else {
+		for _, stream := range self.streams {
+			if pid == stream.pid {
+				if err = stream.handleTSPacket(start, iskeyframe, payload); err != nil {
+					return
+				}
+				break
+			}
+		}
+	}
+
+	return
+}
+
+func (self *Stream) addPacket(payload []byte, timedelta time.Duration) {
+	dts := self.dts
+	pts := self.pts
+	if dts == 0 {
+		dts = pts
+	}
+
+	demuxer := self.demuxer
+	pkt := av.Packet{
+		Idx:        int8(self.idx),
+		IsKeyFrame: self.iskeyframe,
+		Time:       dts + timedelta,
+		Data:       payload,
+	}
+	if pts != dts {
+		pkt.CompositionTime = pts - dts
+	}
+	demuxer.pkts = append(demuxer.pkts, pkt)
+}
+
+func (self *Stream) payloadEnd() (n int, err error) {
+	payload := self.data
+	if payload == nil {
+		return
+	}
+	if self.datalen != 0 && len(payload) != self.datalen {
+		err = fmt.Errorf("ts: packet size mismatch size=%d correct=%d", len(payload), self.datalen)
+		return
+	}
+	self.data = nil
+
+	switch self.streamType {
+	case tsio.ElementaryStreamTypeAdtsAAC:
+		var config aacparser.MPEG4AudioConfig
+
+		delta := time.Duration(0)
+		for len(payload) > 0 {
+			var hdrlen, framelen, samples int
+			if config, hdrlen, framelen, samples, err = aacparser.ParseADTSHeader(payload); err != nil {
+				return
+			}
+			if self.CodecData == nil {
+				if self.CodecData, err = aacparser.NewCodecDataFromMPEG4AudioConfig(config); err != nil {
+					return
+				}
+			}
+			self.addPacket(payload[hdrlen:framelen], delta)
+			n++
+			delta += time.Duration(samples) * time.Second / time.Duration(config.SampleRate)
+			payload = payload[framelen:]
+		}
+
+	case tsio.ElementaryStreamTypeH264:
+		nalus, _ := h264parser.SplitNALUs(payload)
+		var sps, pps []byte
+		for _, nalu := range nalus {
+			if len(nalu) > 0 {
+				naltype := nalu[0] & 0x1f
+				switch {
+				case naltype == 7:
+					sps = nalu
+				case naltype == 8:
+					pps = nalu
+				case h264parser.IsDataNALU(nalu):
+					// raw nalu to avcc
+					b := make([]byte, 4+len(nalu))
+					pio.PutU32BE(b[0:4], uint32(len(nalu)))
+					copy(b[4:], nalu)
+					self.addPacket(b, time.Duration(0))
+					n++
+				}
+			}
+		}
+
+		if self.CodecData == nil && len(sps) > 0 && len(pps) > 0 {
+			if self.CodecData, err = h264parser.NewCodecDataFromSPSAndPPS(sps, pps); err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func (self *Stream) handleTSPacket(start bool, iskeyframe bool, payload []byte) (err error) {
+	if start {
+		if _, err = self.payloadEnd(); err != nil {
+			return
+		}
+		var hdrlen int
+		if hdrlen, _, self.datalen, self.pts, self.dts, err = tsio.ParsePESHeader(payload); err != nil {
+			return
+		}
+		self.iskeyframe = iskeyframe
+		if self.datalen == 0 {
+			self.data = make([]byte, 0, 4096)
+		} else {
+			self.data = make([]byte, 0, self.datalen)
+		}
+		self.data = append(self.data, payload[hdrlen:]...)
+	} else {
+		self.data = append(self.data, payload...)
+	}
+	return
+}

format/ts/handler.go

@@ -0,0 +1,26 @@
+package ts
+
+import (
+	"io"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/av/avutil"
+)
+
+func Handler(h *avutil.RegisterHandler) {
+	h.Ext = ".ts"
+
+	h.Probe = func(b []byte) bool {
+		return b[0] == 0x47 && b[188] == 0x47
+	}
+
+	h.ReaderDemuxer = func(r io.Reader) av.Demuxer {
+		return NewDemuxer(r)
+	}
+
+	h.WriterMuxer = func(w io.Writer) av.Muxer {
+		return NewMuxer(w)
+	}
+
+	h.CodecTypes = CodecTypes
+}

format/ts/muxer.go

@@ -0,0 +1,206 @@
+package ts
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/codec/aacparser"
+	"github.com/deepch/vdk/codec/h264parser"
+	"github.com/deepch/vdk/format/ts/tsio"
+)
+
+var CodecTypes = []av.CodecType{av.H264, av.AAC}
+
+type Muxer struct {
+	w                        io.Writer
+	streams                  []*Stream
+	PaddingToMakeCounterCont bool
+
+	psidata []byte
+	peshdr  []byte
+	tshdr   []byte
+	adtshdr []byte
+	datav   [][]byte
+	nalus   [][]byte
+
+	tswpat, tswpmt *tsio.TSWriter
+}
+
+func NewMuxer(w io.Writer) *Muxer {
+	return &Muxer{
+		w:       w,
+		psidata: make([]byte, 188),
+		peshdr:  make([]byte, tsio.MaxPESHeaderLength),
+		tshdr:   make([]byte, tsio.MaxTSHeaderLength),
+		adtshdr: make([]byte, aacparser.ADTSHeaderLength),
+		nalus:   make([][]byte, 16),
+		datav:   make([][]byte, 16),
+		tswpmt:  tsio.NewTSWriter(tsio.PMT_PID),
+		tswpat:  tsio.NewTSWriter(tsio.PAT_PID),
+	}
+}
+
+func (self *Muxer) newStream(codec av.CodecData) (err error) {
+	ok := false
+	for _, c := range CodecTypes {
+		if codec.Type() == c {
+			ok = true
+			break
+		}
+	}
+	if !ok {
+		err = fmt.Errorf("ts: codec type=%s is not supported", codec.Type())
+		return
+	}
+
+	pid := uint16(len(self.streams) + 0x100)
+	stream := &Stream{
+		muxer:     self,
+		CodecData: codec,
+		pid:       pid,
+		tsw:       tsio.NewTSWriter(pid),
+	}
+	self.streams = append(self.streams, stream)
+	return
+}
+
+func (self *Muxer) writePaddingTSPackets(tsw *tsio.TSWriter) (err error) {
+	for tsw.ContinuityCounter&0xf != 0x0 {
+		if err = tsw.WritePackets(self.w, self.datav[:0], 0, false, true); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (self *Muxer) WriteTrailer() (err error) {
+	if self.PaddingToMakeCounterCont {
+		for _, stream := range self.streams {
+			if err = self.writePaddingTSPackets(stream.tsw); err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+func (self *Muxer) SetWriter(w io.Writer) {
+	self.w = w
+	return
+}
+
+func (self *Muxer) WritePATPMT() (err error) {
+	pat := tsio.PAT{
+		Entries: []tsio.PATEntry{
+			{ProgramNumber: 1, ProgramMapPID: tsio.PMT_PID},
+		},
+	}
+	patlen := pat.Marshal(self.psidata[tsio.PSIHeaderLength:])
+	n := tsio.FillPSI(self.psidata, tsio.TableIdPAT, tsio.TableExtPAT, patlen)
+	self.datav[0] = self.psidata[:n]
+	if err = self.tswpat.WritePackets(self.w, self.datav[:1], 0, false, true); err != nil {
+		return
+	}
+
+	var elemStreams []tsio.ElementaryStreamInfo
+	for _, stream := range self.streams {
+		switch stream.Type() {
+		case av.AAC:
+			elemStreams = append(elemStreams, tsio.ElementaryStreamInfo{
+				StreamType:    tsio.ElementaryStreamTypeAdtsAAC,
+				ElementaryPID: stream.pid,
+			})
+		case av.H264:
+			elemStreams = append(elemStreams, tsio.ElementaryStreamInfo{
+				StreamType:    tsio.ElementaryStreamTypeH264,
+				ElementaryPID: stream.pid,
+			})
+		}
+	}
+
+	pmt := tsio.PMT{
+		PCRPID:                0x100,
+		ElementaryStreamInfos: elemStreams,
+	}
+	pmtlen := pmt.Len()
+	if pmtlen+tsio.PSIHeaderLength > len(self.psidata) {
+		err = fmt.Errorf("ts: pmt too large")
+		return
+	}
+	pmt.Marshal(self.psidata[tsio.PSIHeaderLength:])
+	n = tsio.FillPSI(self.psidata, tsio.TableIdPMT, tsio.TableExtPMT, pmtlen)
+	self.datav[0] = self.psidata[:n]
+	if err = self.tswpmt.WritePackets(self.w, self.datav[:1], 0, false, true); err != nil {
+		return
+	}
+
+	return
+}
+
+func (self *Muxer) WriteHeader(streams []av.CodecData) (err error) {
+	self.streams = []*Stream{}
+	for _, stream := range streams {
+		if err = self.newStream(stream); err != nil {
+			return
+		}
+	}
+
+	if err = self.WritePATPMT(); err != nil {
+		return
+	}
+	return
+}
+
+func (self *Muxer) WritePacket(pkt av.Packet) (err error) {
+	stream := self.streams[pkt.Idx]
+	pkt.Time += time.Second
+
+	switch stream.Type() {
+	case av.AAC:
+		codec := stream.CodecData.(aacparser.CodecData)
+
+		n := tsio.FillPESHeader(self.peshdr, tsio.StreamIdAAC, len(self.adtshdr)+len(pkt.Data), pkt.Time, 0)
+		self.datav[0] = self.peshdr[:n]
+		aacparser.FillADTSHeader(self.adtshdr, codec.Config, 1024, len(pkt.Data))
+		self.datav[1] = self.adtshdr
+		self.datav[2] = pkt.Data
+
+		if err = stream.tsw.WritePackets(self.w, self.datav[:3], pkt.Time, true, false); err != nil {
+			return
+		}
+
+	case av.H264:
+		codec := stream.CodecData.(h264parser.CodecData)
+
+		nalus := self.nalus[:0]
+		if pkt.IsKeyFrame {
+			nalus = append(nalus, codec.SPS())
+			nalus = append(nalus, codec.PPS())
+		}
+		pktnalus, _ := h264parser.SplitNALUs(pkt.Data)
+		for _, nalu := range pktnalus {
+			nalus = append(nalus, nalu)
+		}
+
+		datav := self.datav[:1]
+		for i, nalu := range nalus {
+			if i == 0 {
+				datav = append(datav, h264parser.AUDBytes)
+			} else {
+				datav = append(datav, h264parser.StartCodeBytes)
+			}
+			datav = append(datav, nalu)
+		}
+
+		n := tsio.FillPESHeader(self.peshdr, tsio.StreamIdH264, -1, pkt.Time+pkt.CompositionTime, pkt.Time)
+		datav[0] = self.peshdr[:n]
+
+		if err = stream.tsw.WritePackets(self.w, datav, pkt.Time, pkt.IsKeyFrame, false); err != nil {
+			return
+		}
+	}
+
+	return
+}

format/ts/stream.go

@@ -0,0 +1,27 @@
+package ts
+
+import (
+	"time"
+
+	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/format/ts/tsio"
+)
+
+type Stream struct {
+	av.CodecData
+
+	demuxer *Demuxer
+	muxer   *Muxer
+
+	pid        uint16
+	streamId   uint8
+	streamType uint8
+
+	tsw *tsio.TSWriter
+	idx int
+
+	iskeyframe bool
+	pts, dts   time.Duration
+	data       []byte
+	datalen    int
+}

format/ts/tsio/checksum.go

@@ -0,0 +1,55 @@
+package tsio
+
+var ieeeCrc32Tbl = []uint32{
+	0x00000000, 0xB71DC104, 0x6E3B8209, 0xD926430D, 0xDC760413, 0x6B6BC517,
+	0xB24D861A, 0x0550471E, 0xB8ED0826, 0x0FF0C922, 0xD6D68A2F, 0x61CB4B2B,
+	0x649B0C35, 0xD386CD31, 0x0AA08E3C, 0xBDBD4F38, 0x70DB114C, 0xC7C6D048,
+	0x1EE09345, 0xA9FD5241, 0xACAD155F, 0x1BB0D45B, 0xC2969756, 0x758B5652,
+	0xC836196A, 0x7F2BD86E, 0xA60D9B63, 0x11105A67, 0x14401D79, 0xA35DDC7D,
+	0x7A7B9F70, 0xCD665E74, 0xE0B62398, 0x57ABE29C, 0x8E8DA191, 0x39906095,
+	0x3CC0278B, 0x8BDDE68F, 0x52FBA582, 0xE5E66486, 0x585B2BBE, 0xEF46EABA,
+	0x3660A9B7, 0x817D68B3, 0x842D2FAD, 0x3330EEA9, 0xEA16ADA4, 0x5D0B6CA0,
+	0x906D32D4, 0x2770F3D0, 0xFE56B0DD, 0x494B71D9, 0x4C1B36C7, 0xFB06F7C3,
+	0x2220B4CE, 0x953D75CA, 0x28803AF2, 0x9F9DFBF6, 0x46BBB8FB, 0xF1A679FF,
+	0xF4F63EE1, 0x43EBFFE5, 0x9ACDBCE8, 0x2DD07DEC, 0x77708634, 0xC06D4730,
+	0x194B043D, 0xAE56C539, 0xAB068227, 0x1C1B4323, 0xC53D002E, 0x7220C12A,
+	0xCF9D8E12, 0x78804F16, 0xA1A60C1B, 0x16BBCD1F, 0x13EB8A01, 0xA4F64B05,
+	0x7DD00808, 0xCACDC90C, 0x07AB9778, 0xB0B6567C, 0x69901571, 0xDE8DD475,
+	0xDBDD936B, 0x6CC0526F, 0xB5E61162, 0x02FBD066, 0xBF469F5E, 0x085B5E5A,
+	0xD17D1D57, 0x6660DC53, 0x63309B4D, 0xD42D5A49, 0x0D0B1944, 0xBA16D840,
+	0x97C6A5AC, 0x20DB64A8, 0xF9FD27A5, 0x4EE0E6A1, 0x4BB0A1BF, 0xFCAD60BB,
+	0x258B23B6, 0x9296E2B2, 0x2F2BAD8A, 0x98366C8E, 0x41102F83, 0xF60DEE87,
+	0xF35DA999, 0x4440689D, 0x9D662B90, 0x2A7BEA94, 0xE71DB4E0, 0x500075E4,
+	0x892636E9, 0x3E3BF7ED, 0x3B6BB0F3, 0x8C7671F7, 0x555032FA, 0xE24DF3FE,
+	0x5FF0BCC6, 0xE8ED7DC2, 0x31CB3ECF, 0x86D6FFCB, 0x8386B8D5, 0x349B79D1,
+	0xEDBD3ADC, 0x5AA0FBD8, 0xEEE00C69, 0x59FDCD6D, 0x80DB8E60, 0x37C64F64,
+	0x3296087A, 0x858BC97E, 0x5CAD8A73, 0xEBB04B77, 0x560D044F, 0xE110C54B,
+	0x38368646, 0x8F2B4742, 0x8A7B005C, 0x3D66C158, 0xE4408255, 0x535D4351,
+	0x9E3B1D25, 0x2926DC21, 0xF0009F2C, 0x471D5E28, 0x424D1936, 0xF550D832,
+	0x2C769B3F, 0x9B6B5A3B, 0x26D61503, 0x91CBD407, 0x48ED970A, 0xFFF0560E,
+	0xFAA01110, 0x4DBDD014, 0x949B9319, 0x2386521D, 0x0E562FF1, 0xB94BEEF5,
+	0x606DADF8, 0xD7706CFC, 0xD2202BE2, 0x653DEAE6, 0xBC1BA9EB, 0x0B0668EF,
+	0xB6BB27D7, 0x01A6E6D3, 0xD880A5DE, 0x6F9D64DA, 0x6ACD23C4, 0xDDD0E2C0,
+	0x04F6A1CD, 0xB3EB60C9, 0x7E8D3EBD, 0xC990FFB9, 0x10B6BCB4, 0xA7AB7DB0,
+	0xA2FB3AAE, 0x15E6FBAA, 0xCCC0B8A7, 0x7BDD79A3, 0xC660369B, 0x717DF79F,
+	0xA85BB492, 0x1F467596, 0x1A163288, 0xAD0BF38C, 0x742DB081, 0xC3307185,
+	0x99908A5D, 0x2E8D4B59, 0xF7AB0854, 0x40B6C950, 0x45E68E4E, 0xF2FB4F4A,
+	0x2BDD0C47, 0x9CC0CD43, 0x217D827B, 0x9660437F, 0x4F460072, 0xF85BC176,
+	0xFD0B8668, 0x4A16476C, 0x93300461, 0x242DC565, 0xE94B9B11, 0x5E565A15,
+	0x87701918, 0x306DD81C, 0x353D9F02, 0x82205E06, 0x5B061D0B, 0xEC1BDC0F,
+	0x51A69337, 0xE6BB5233, 0x3F9D113E, 0x8880D03A, 0x8DD09724, 0x3ACD5620,
+	0xE3EB152D, 0x54F6D429, 0x7926A9C5, 0xCE3B68C1, 0x171D2BCC, 0xA000EAC8,
+	0xA550ADD6, 0x124D6CD2, 0xCB6B2FDF, 0x7C76EEDB, 0xC1CBA1E3, 0x76D660E7,
+	0xAFF023EA, 0x18EDE2EE, 0x1DBDA5F0, 0xAAA064F4, 0x738627F9, 0xC49BE6FD,
+	0x09FDB889, 0xBEE0798D, 0x67C63A80, 0xD0DBFB84, 0xD58BBC9A, 0x62967D9E,
+	0xBBB03E93, 0x0CADFF97, 0xB110B0AF, 0x060D71AB, 0xDF2B32A6, 0x6836F3A2,
+	0x6D66B4BC, 0xDA7B75B8, 0x035D36B5, 0xB440F7B1, 0x00000001,
+}
+
+func calcCRC32(crc uint32, data []byte) uint32 {
+	for _, b := range data {
+		crc = ieeeCrc32Tbl[b^byte(crc)] ^ (crc >> 8)
+	}
+	return crc
+}
+

format/ts/tsio/tsio.go

@@ -0,0 +1,589 @@
+package tsio
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/deepch/vdk/utils/bits/pio"
+)
+
+const (
+	StreamIdH264 = 0xe0
+	StreamIdAAC  = 0xc0
+)
+
+const (
+	PAT_PID = 0
+	PMT_PID = 0x1000
+)
+
+const TableIdPMT = 2
+const TableExtPMT = 1
+
+const TableIdPAT = 0
+const TableExtPAT = 1
+
+const MaxPESHeaderLength = 19
+const MaxTSHeaderLength = 12
+
+var ErrPESHeader = fmt.Errorf("invalid PES header")
+var ErrPSIHeader = fmt.Errorf("invalid PSI header")
+var ErrParsePMT = fmt.Errorf("invalid PMT")
+var ErrParsePAT = fmt.Errorf("invalid PAT")
+
+const (
+	ElementaryStreamTypeH264    = 0x1B
+	ElementaryStreamTypeAdtsAAC = 0x0F
+)
+
+type PATEntry struct {
+	ProgramNumber uint16
+	NetworkPID    uint16
+	ProgramMapPID uint16
+}
+
+type PAT struct {
+	Entries []PATEntry
+}
+
+func (self PAT) Len() (n int) {
+	return len(self.Entries) * 4
+}
+
+func (self PAT) Marshal(b []byte) (n int) {
+	for _, entry := range self.Entries {
+		pio.PutU16BE(b[n:], entry.ProgramNumber)
+		n += 2
+		if entry.ProgramNumber == 0 {
+			pio.PutU16BE(b[n:], entry.NetworkPID&0x1fff|7<<13)
+			n += 2
+		} else {
+			pio.PutU16BE(b[n:], entry.ProgramMapPID&0x1fff|7<<13)
+			n += 2
+		}
+	}
+	return
+}
+
+func (self *PAT) Unmarshal(b []byte) (n int, err error) {
+	for n < len(b) {
+		if n+4 <= len(b) {
+			var entry PATEntry
+			entry.ProgramNumber = pio.U16BE(b[n:])
+			n += 2
+			if entry.ProgramNumber == 0 {
+				entry.NetworkPID = pio.U16BE(b[n:]) & 0x1fff
+				n += 2
+			} else {
+				entry.ProgramMapPID = pio.U16BE(b[n:]) & 0x1fff
+				n += 2
+			}
+			self.Entries = append(self.Entries, entry)
+		} else {
+			break
+		}
+	}
+	if n < len(b) {
+		err = ErrParsePAT
+		return
+	}
+	return
+}
+
+type Descriptor struct {
+	Tag  uint8
+	Data []byte
+}
+
+type ElementaryStreamInfo struct {
+	StreamType    uint8
+	ElementaryPID uint16
+	Descriptors   []Descriptor
+}
+
+type PMT struct {
+	PCRPID                uint16
+	ProgramDescriptors    []Descriptor
+	ElementaryStreamInfos []ElementaryStreamInfo
+}
+
+func (self PMT) Len() (n int) {
+	// 111(3)
+	// PCRPID(13)
+	n += 2
+
+	// desclen(16)
+	n += 2
+
+	for _, desc := range self.ProgramDescriptors {
+		n += 2 + len(desc.Data)
+	}
+
+	for _, info := range self.ElementaryStreamInfos {
+		// streamType
+		n += 1
+
+		// Reserved(3)
+		// Elementary PID(13)
+		n += 2
+
+		// Reserved(6)
+		// ES Info length length(10)
+		n += 2
+
+		for _, desc := range info.Descriptors {
+			n += 2 + len(desc.Data)
+		}
+	}
+
+	return
+}
+
+func (self PMT) fillDescs(b []byte, descs []Descriptor) (n int) {
+	for _, desc := range descs {
+		b[n] = desc.Tag
+		n++
+		b[n] = uint8(len(desc.Data))
+		n++
+		copy(b[n:], desc.Data)
+		n += len(desc.Data)
+	}
+	return
+}
+
+func (self PMT) Marshal(b []byte) (n int) {
+	// 111(3)
+	// PCRPID(13)
+	pio.PutU16BE(b[n:], self.PCRPID|7<<13)
+	n += 2
+
+	hold := n
+	n += 2
+	pos := n
+	n += self.fillDescs(b[n:], self.ProgramDescriptors)
+	desclen := n - pos
+	pio.PutU16BE(b[hold:], uint16(desclen)|0xf<<12)
+
+	for _, info := range self.ElementaryStreamInfos {
+		b[n] = info.StreamType
+		n++
+
+		// Reserved(3)
+		// Elementary PID(13)
+		pio.PutU16BE(b[n:], info.ElementaryPID|7<<13)
+		n += 2
+
+		hold := n
+		n += 2
+		pos := n
+		n += self.fillDescs(b[n:], info.Descriptors)
+		desclen := n - pos
+		pio.PutU16BE(b[hold:], uint16(desclen)|0x3c<<10)
+	}
+
+	return
+}
+
+func (self PMT) parseDescs(b []byte) (descs []Descriptor, err error) {
+	n := 0
+	for n < len(b) {
+		if n+2 <= len(b) {
+			desc := Descriptor{}
+			desc.Tag = b[n]
+			desc.Data = make([]byte, b[n+1])
+			n += 2
+			if n+len(desc.Data) < len(b) {
+				copy(desc.Data, b[n:])
+				descs = append(descs, desc)
+				n += len(desc.Data)
+			} else {
+				break
+			}
+		} else {
+			break
+		}
+	}
+	if n < len(b) {
+		err = ErrParsePMT
+		return
+	}
+	return
+}
+
+func (self *PMT) Unmarshal(b []byte) (n int, err error) {
+	if len(b) < n+4 {
+		err = ErrParsePMT
+		return
+	}
+
+	// 111(3)
+	// PCRPID(13)
+	self.PCRPID = pio.U16BE(b[0:2]) & 0x1fff
+	n += 2
+
+	// Reserved(4)=0xf
+	// Reserved(2)=0x0
+	// Program info length(10)
+	desclen := int(pio.U16BE(b[2:4]) & 0x3ff)
+	n += 2
+
+	if desclen > 0 {
+		if len(b) < n+desclen {
+			err = ErrParsePMT
+			return
+		}
+		if self.ProgramDescriptors, err = self.parseDescs(b[n : n+desclen]); err != nil {
+			return
+		}
+		n += desclen
+	}
+
+	for n < len(b) {
+		if len(b) < n+5 {
+			err = ErrParsePMT
+			return
+		}
+
+		var info ElementaryStreamInfo
+		info.StreamType = b[n]
+		n++
+
+		// Reserved(3)
+		// Elementary PID(13)
+		info.ElementaryPID = pio.U16BE(b[n:]) & 0x1fff
+		n += 2
+
+		// Reserved(6)
+		// ES Info length(10)
+		desclen := int(pio.U16BE(b[n:]) & 0x3ff)
+		n += 2
+
+		if desclen > 0 {
+			if len(b) < n+desclen {
+				err = ErrParsePMT
+				return
+			}
+			if info.Descriptors, err = self.parseDescs(b[n : n+desclen]); err != nil {
+				return
+			}
+			n += desclen
+		}
+
+		self.ElementaryStreamInfos = append(self.ElementaryStreamInfos, info)
+	}
+
+	return
+}
+
+func ParsePSI(h []byte) (tableid uint8, tableext uint16, hdrlen int, datalen int, err error) {
+	if len(h) < 8 {
+		err = ErrPSIHeader
+		return
+	}
+
+	// pointer(8)
+	pointer := h[0]
+	hdrlen++
+	if pointer > 0 {
+		hdrlen += int(pointer)
+		if len(h) < hdrlen {
+			err = ErrPSIHeader
+			return
+		}
+	}
+
+	if len(h) < hdrlen+12 {
+		err = ErrPSIHeader
+		return
+	}
+
+	// table_id(8)
+	tableid = h[hdrlen]
+	hdrlen++
+
+	// section_syntax_indicator(1)=1,private_bit(1)=0,reserved(2)=3,unused(2)=0,section_length(10)
+	datalen = int(pio.U16BE(h[hdrlen:]))&0x3ff - 9
+	hdrlen += 2
+
+	if datalen < 0 {
+		err = ErrPSIHeader
+		return
+	}
+
+	// Table ID extension(16)
+	tableext = pio.U16BE(h[hdrlen:])
+	hdrlen += 2
+
+	// resverd(2)=3
+	// version(5)
+	// Current_next_indicator(1)
+	hdrlen++
+
+	// section_number(8)
+	hdrlen++
+
+	// last_section_number(8)
+	hdrlen++
+
+	// data
+
+	// crc(32)
+
+	return
+}
+
+const PSIHeaderLength = 9
+
+func FillPSI(h []byte, tableid uint8, tableext uint16, datalen int) (n int) {
+	// pointer(8)
+	h[n] = 0
+	n++
+
+	// table_id(8)
+	h[n] = tableid
+	n++
+
+	// section_syntax_indicator(1)=1,private_bit(1)=0,reserved(2)=3,unused(2)=0,section_length(10)
+	pio.PutU16BE(h[n:], uint16(0xa<<12|2+3+4+datalen))
+	n += 2
+
+	// Table ID extension(16)
+	pio.PutU16BE(h[n:], tableext)
+	n += 2
+
+	// resverd(2)=3,version(5)=0,Current_next_indicator(1)=1
+	h[n] = 0x3<<6 | 1
+	n++
+
+	// section_number(8)
+	h[n] = 0
+	n++
+
+	// last_section_number(8)
+	h[n] = 0
+	n++
+
+	n += datalen
+
+	crc := calcCRC32(0xffffffff, h[1:n])
+	pio.PutU32LE(h[n:], crc)
+	n += 4
+
+	return
+}
+
+func TimeToPCR(tm time.Duration) (pcr uint64) {
+	// base(33)+resverd(6)+ext(9)
+	ts := uint64(tm * PCR_HZ / time.Second)
+	base := ts / 300
+	ext := ts % 300
+	pcr = base<<15 | 0x3f<<9 | ext
+	return
+}
+
+func PCRToTime(pcr uint64) (tm time.Duration) {
+	base := pcr >> 15
+	ext := pcr & 0x1ff
+	ts := base*300 + ext
+	tm = time.Duration(ts) * time.Second / time.Duration(PCR_HZ)
+	return
+}
+
+func TimeToTs(tm time.Duration) (v uint64) {
+	ts := uint64(tm * PTS_HZ / time.Second)
+	// 0010	PTS 32..30 1	PTS 29..15 1 PTS 14..00 1
+	v = ((ts>>30)&0x7)<<33 | ((ts>>15)&0x7fff)<<17 | (ts&0x7fff)<<1 | 0x100010001
+	return
+}
+
+func TsToTime(v uint64) (tm time.Duration) {
+	// 0010	PTS 32..30 1	PTS 29..15 1 PTS 14..00 1
+	ts := (((v >> 33) & 0x7) << 30) | (((v >> 17) & 0x7fff) << 15) | ((v >> 1) & 0x7fff)
+	tm = time.Duration(ts) * time.Second / time.Duration(PTS_HZ)
+	return
+}
+
+const (
+	PTS_HZ = 90000
+	PCR_HZ = 27000000
+)
+
+func ParsePESHeader(h []byte) (hdrlen int, streamid uint8, datalen int, pts, dts time.Duration, err error) {
+	if h[0] != 0 || h[1] != 0 || h[2] != 1 {
+		err = ErrPESHeader
+		return
+	}
+	streamid = h[3]
+
+	flags := h[7]
+	hdrlen = int(h[8]) + 9
+
+	datalen = int(pio.U16BE(h[4:6]))
+	if datalen > 0 {
+		datalen -= int(h[8]) + 3
+	}
+
+	const PTS = 1 << 7
+	const DTS = 1 << 6
+
+	if flags&PTS != 0 {
+		if len(h) < 14 {
+			err = ErrPESHeader
+			return
+		}
+		pts = TsToTime(pio.U40BE(h[9:14]))
+		if flags&DTS != 0 {
+			if len(h) < 19 {
+				err = ErrPESHeader
+				return
+			}
+			dts = TsToTime(pio.U40BE(h[14:19]))
+		}
+	}
+
+	return
+}
+
+func FillPESHeader(h []byte, streamid uint8, datalen int, pts, dts time.Duration) (n int) {
+	h[0] = 0
+	h[1] = 0
+	h[2] = 1
+	h[3] = streamid
+
+	const PTS = 1 << 7
+	const DTS = 1 << 6
+
+	var flags uint8
+	if pts != 0 {
+		flags |= PTS
+		if dts != 0 {
+			flags |= DTS
+		}
+	}
+
+	if flags&PTS != 0 {
+		n += 5
+	}
+	if flags&DTS != 0 {
+		n += 5
+	}
+
+	// packet_length(16) if zero then variable length
+	// Specifies the number of bytes remaining in the packet after this field. Can be zero.
+	// If the PES packet length is set to zero, the PES packet can be of any length.
+	// A value of zero for the PES packet length can be used only when the PES packet payload is a **video** elementary stream.
+	var pktlen uint16
+	if datalen >= 0 {
+		pktlen = uint16(datalen + n + 3)
+	}
+	pio.PutU16BE(h[4:6], pktlen)
+
+	h[6] = 2<<6 | 1 // resverd(6,2)=2,original_or_copy(0,1)=1
+	h[7] = flags
+	h[8] = uint8(n)
+
+	// pts(40)?
+	// dts(40)?
+	if flags&PTS != 0 {
+		if flags&DTS != 0 {
+			pio.PutU40BE(h[9:14], TimeToTs(pts)|3<<36)
+			pio.PutU40BE(h[14:19], TimeToTs(dts)|1<<36)
+		} else {
+			pio.PutU40BE(h[9:14], TimeToTs(pts)|2<<36)
+		}
+	}
+
+	n += 9
+	return
+}
+
+type TSWriter struct {
+	w                 io.Writer
+	ContinuityCounter uint
+	tshdr             []byte
+}
+
+func NewTSWriter(pid uint16) *TSWriter {
+	w := &TSWriter{}
+	w.tshdr = make([]byte, 188)
+	w.tshdr[0] = 0x47
+	pio.PutU16BE(w.tshdr[1:3], pid&0x1fff)
+	for i := 6; i < 188; i++ {
+		w.tshdr[i] = 0xff
+	}
+	return w
+}
+
+func (self *TSWriter) WritePackets(w io.Writer, datav [][]byte, pcr time.Duration, sync bool, paddata bool) (err error) {
+	datavlen := pio.VecLen(datav)
+	writev := make([][]byte, len(datav))
+	writepos := 0
+
+	for writepos < datavlen {
+		self.tshdr[1] = self.tshdr[1] & 0x1f
+		self.tshdr[3] = byte(self.ContinuityCounter)&0xf | 0x30
+		self.tshdr[5] = 0 // flags
+		hdrlen := 6
+		self.ContinuityCounter++
+
+		if writepos == 0 {
+			self.tshdr[1] = 0x40 | self.tshdr[1] // Payload Unit Start Indicator
+			if pcr != 0 {
+				hdrlen += 6
+				self.tshdr[5] = 0x10 | self.tshdr[5] // PCR flag (Discontinuity indicator 0x80)
+				pio.PutU48BE(self.tshdr[6:12], TimeToPCR(pcr))
+			}
+			if sync {
+				self.tshdr[5] = 0x40 | self.tshdr[5] // Random Access indicator
+			}
+		}
+
+		padtail := 0
+		end := writepos + 188 - hdrlen
+		if end > datavlen {
+			if paddata {
+				padtail = end - datavlen
+			} else {
+				hdrlen += end - datavlen
+			}
+			end = datavlen
+		}
+		n := pio.VecSliceTo(datav, writev, writepos, end)
+
+		self.tshdr[4] = byte(hdrlen) - 5 // length
+		if _, err = w.Write(self.tshdr[:hdrlen]); err != nil {
+			return
+		}
+		for i := 0; i < n; i++ {
+			if _, err = w.Write(writev[i]); err != nil {
+				return
+			}
+		}
+		if padtail > 0 {
+			if _, err = w.Write(self.tshdr[188-padtail : 188]); err != nil {
+				return
+			}
+		}
+
+		writepos = end
+	}
+
+	return
+}
+
+func ParseTSHeader(tshdr []byte) (pid uint16, start bool, iskeyframe bool, hdrlen int, err error) {
+	// https://en.wikipedia.org/wiki/MPEG_transport_stream
+	if tshdr[0] != 0x47 {
+		err = fmt.Errorf("tshdr sync invalid")
+		return
+	}
+	pid = uint16((tshdr[1]&0x1f))<<8 | uint16(tshdr[2])
+	start = tshdr[1]&0x40 != 0
+	hdrlen += 4
+	if tshdr[3]&0x20 != 0 {
+		hdrlen += int(tshdr[4]) + 1
+		iskeyframe = tshdr[5]&0x40 != 0
+	}
+	return
+}

utils/bits/bits.go

@@ -0,0 +1,118 @@
+package bits
+
+import (
+	"io"
+)
+
+type Reader struct {
+	R    io.Reader
+	n    int
+	bits uint64
+}
+
+func (self *Reader) ReadBits64(n int) (bits uint64, err error) {
+	if self.n < n {
+		var b [8]byte
+		var got int
+		want := (n - self.n + 7) / 8
+		if got, err = self.R.Read(b[:want]); err != nil {
+			return
+		}
+		if got < want {
+			err = io.EOF
+			return
+		}
+		for i := 0; i < got; i++ {
+			self.bits <<= 8
+			self.bits |= uint64(b[i])
+		}
+		self.n += got * 8
+	}
+	bits = self.bits >> uint(self.n-n)
+	self.bits ^= bits << uint(self.n-n)
+	self.n -= n
+	return
+}
+
+func (self *Reader) ReadBits(n int) (bits uint, err error) {
+	var bits64 uint64
+	if bits64, err = self.ReadBits64(n); err != nil {
+		return
+	}
+	bits = uint(bits64)
+	return
+}
+
+func (self *Reader) Read(p []byte) (n int, err error) {
+	for n < len(p) {
+		want := 8
+		if len(p)-n < want {
+			want = len(p) - n
+		}
+		var bits uint64
+		if bits, err = self.ReadBits64(want * 8); err != nil {
+			break
+		}
+		for i := 0; i < want; i++ {
+			p[n+i] = byte(bits >> uint((want-i-1)*8))
+		}
+		n += want
+	}
+	return
+}
+
+type Writer struct {
+	W    io.Writer
+	n    int
+	bits uint64
+}
+
+func (self *Writer) WriteBits64(bits uint64, n int) (err error) {
+	if self.n+n > 64 {
+		move := uint(64 - self.n)
+		mask := bits >> move
+		self.bits = (self.bits << move) | mask
+		self.n = 64
+		if err = self.FlushBits(); err != nil {
+			return
+		}
+		n -= int(move)
+		bits ^= (mask << move)
+	}
+	self.bits = (self.bits << uint(n)) | bits
+	self.n += n
+	return
+}
+
+func (self *Writer) WriteBits(bits uint, n int) (err error) {
+	return self.WriteBits64(uint64(bits), n)
+}
+
+func (self *Writer) Write(p []byte) (n int, err error) {
+	for n < len(p) {
+		if err = self.WriteBits64(uint64(p[n]), 8); err != nil {
+			return
+		}
+		n++
+	}
+	return
+}
+
+func (self *Writer) FlushBits() (err error) {
+	if self.n > 0 {
+		var b [8]byte
+		bits := self.bits
+		if self.n%8 != 0 {
+			bits <<= uint(8 - (self.n % 8))
+		}
+		want := (self.n + 7) / 8
+		for i := 0; i < want; i++ {
+			b[i] = byte(bits >> uint((want-i-1)*8))
+		}
+		if _, err = self.W.Write(b[:want]); err != nil {
+			return
+		}
+		self.n = 0
+	}
+	return
+}

utils/bits/bits_test.go

@@ -0,0 +1,51 @@
+package bits
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestBits(t *testing.T) {
+	rdata := []byte{0xf3, 0xb3, 0x45, 0x60}
+	rbuf := bytes.NewReader(rdata[:])
+	r := &Reader{R: rbuf}
+	var u32 uint
+	if u32, _ = r.ReadBits(4); u32 != 0xf {
+		t.FailNow()
+	}
+	if u32, _ = r.ReadBits(4); u32 != 0x3 {
+		t.FailNow()
+	}
+	if u32, _ = r.ReadBits(2); u32 != 0x2 {
+		t.FailNow()
+	}
+	if u32, _ = r.ReadBits(2); u32 != 0x3 {
+		t.FailNow()
+	}
+	b := make([]byte, 2)
+	if r.Read(b); b[0] != 0x34 || b[1] != 0x56 {
+		t.FailNow()
+	}
+
+	wbuf := &bytes.Buffer{}
+	w := &Writer{W: wbuf}
+	w.WriteBits(0xf, 4)
+	w.WriteBits(0x3, 4)
+	w.WriteBits(0x2, 2)
+	w.WriteBits(0x3, 2)
+	n, _ := w.Write([]byte{0x34, 0x56})
+	if n != 2 {
+		t.FailNow()
+	}
+	w.FlushBits()
+	wdata := wbuf.Bytes()
+	if wdata[0] != 0xf3 || wdata[1] != 0xb3 || wdata[2] != 0x45 || wdata[3] != 0x60 {
+		t.FailNow()
+	}
+
+	b = make([]byte, 8)
+	PutUInt64BE(b, 0x11223344, 32)
+	if b[0] != 0x11 || b[1] != 0x22 || b[2] != 0x33 || b[3] != 0x44 {
+		t.FailNow()
+	}
+}

utils/bits/bufio/bufio.go

@@ -0,0 +1,23 @@
+package bufio
+
+import (
+	"io"
+)
+
+type Reader struct {
+	buf [][]byte
+	R io.ReadSeeker
+}
+
+func NewReaderSize(r io.ReadSeeker, size int) *Reader {
+	buf := make([]byte, size*2)
+	return &Reader{
+		R: r,
+		buf: [][]byte{buf[0:size], buf[size:]},
+	}
+}
+
+func (self *Reader) ReadAt(b []byte, off int64) (n int, err error) {
+	return
+}
+

utils/bits/golomb_reader.go

@@ -0,0 +1,65 @@
+package bits
+
+import (
+	"io"
+)
+
+type GolombBitReader struct {
+	R    io.Reader
+	buf  [1]byte
+	left byte
+}
+
+func (self *GolombBitReader) ReadBit() (res uint, err error) {
+	if self.left == 0 {
+		if _, err = self.R.Read(self.buf[:]); err != nil {
+			return
+		}
+		self.left = 8
+	}
+	self.left--
+	res = uint(self.buf[0]>>self.left) & 1
+	return
+}
+
+func (self *GolombBitReader) ReadBits(n int) (res uint, err error) {
+	for i := 0; i < n; i++ {
+		var bit uint
+		if bit, err = self.ReadBit(); err != nil {
+			return
+		}
+		res |= bit << uint(n-i-1)
+	}
+	return
+}
+
+func (self *GolombBitReader) ReadExponentialGolombCode() (res uint, err error) {
+	i := 0
+	for {
+		var bit uint
+		if bit, err = self.ReadBit(); err != nil {
+			return
+		}
+		if !(bit == 0 && i < 32) {
+			break
+		}
+		i++
+	}
+	if res, err = self.ReadBits(i); err != nil {
+		return
+	}
+	res += (1 << uint(i)) - 1
+	return
+}
+
+func (self *GolombBitReader) ReadSE() (res uint, err error) {
+	if res, err = self.ReadExponentialGolombCode(); err != nil {
+		return
+	}
+	if res&0x01 != 0 {
+		res = (res + 1) / 2
+	} else {
+		res = -res / 2
+	}
+	return
+}

utils/bits/pio/pio.go

@@ -0,0 +1,5 @@
+
+package pio
+
+var RecommendBufioSize = 1024*64
+

utils/bits/pio/reader.go

@@ -0,0 +1,91 @@
+
+package pio
+
+func U8(b []byte) (i uint8) {
+	return b[0]
+}
+
+func U16BE(b []byte) (i uint16) {
+	i = uint16(b[0])
+	i <<= 8; i |= uint16(b[1])
+	return
+}
+
+func I16BE(b []byte) (i int16) {
+	i = int16(b[0])
+	i <<= 8; i |= int16(b[1])
+	return
+}
+
+func I24BE(b []byte) (i int32) {
+	i = int32(int8(b[0]))
+	i <<= 8; i |= int32(b[1])
+	i <<= 8; i |= int32(b[2])
+	return
+}
+
+func U24BE(b []byte) (i uint32) {
+	i = uint32(b[0])
+	i <<= 8; i |= uint32(b[1])
+	i <<= 8; i |= uint32(b[2])
+	return
+}
+
+func I32BE(b []byte) (i int32) {
+	i = int32(int8(b[0]))
+	i <<= 8; i |= int32(b[1])
+	i <<= 8; i |= int32(b[2])
+	i <<= 8; i |= int32(b[3])
+	return
+}
+
+func U32LE(b []byte) (i uint32) {
+	i = uint32(b[3])
+	i <<= 8; i |= uint32(b[2])
+	i <<= 8; i |= uint32(b[1])
+	i <<= 8; i |= uint32(b[0])
+	return
+}
+
+func U32BE(b []byte) (i uint32) {
+	i = uint32(b[0])
+	i <<= 8; i |= uint32(b[1])
+	i <<= 8; i |= uint32(b[2])
+	i <<= 8; i |= uint32(b[3])
+	return
+}
+
+func U40BE(b []byte) (i uint64) {
+	i = uint64(b[0])
+	i <<= 8; i |= uint64(b[1])
+	i <<= 8; i |= uint64(b[2])
+	i <<= 8; i |= uint64(b[3])
+	i <<= 8; i |= uint64(b[4])
+	return
+}
+
+func U64BE(b []byte) (i uint64) {
+	i = uint64(b[0])
+	i <<= 8; i |= uint64(b[1])
+	i <<= 8; i |= uint64(b[2])
+	i <<= 8; i |= uint64(b[3])
+	i <<= 8; i |= uint64(b[4])
+	i <<= 8; i |= uint64(b[5])
+	i <<= 8; i |= uint64(b[6])
+	i <<= 8; i |= uint64(b[7])
+	return
+}
+
+func I64BE(b []byte) (i int64) {
+	i = int64(int8(b[0]))
+	i <<= 8; i |= int64(b[1])
+	i <<= 8; i |= int64(b[2])
+	i <<= 8; i |= int64(b[3])
+	i <<= 8; i |= int64(b[4])
+	i <<= 8; i |= int64(b[5])
+	i <<= 8; i |= int64(b[6])
+	i <<= 8; i |= int64(b[7])
+	return
+}
+
+

utils/bits/pio/vec.go

@@ -0,0 +1,69 @@
+package pio
+
+func VecLen(vec [][]byte) (n int) {
+	for _, b := range vec {
+		n += len(b)
+	}
+	return
+}
+
+func VecSliceTo(in [][]byte, out [][]byte, s int, e int) (n int) {
+	if s < 0 {
+		s = 0
+	}
+
+	if e >= 0 && e < s {
+		panic("pio: VecSlice start > end")
+	}
+
+	i := 0
+	off := 0
+	for s > 0 && i < len(in) {
+		left := len(in[i])
+		read := s
+		if left < read {
+			read = left
+		}
+		left -= read
+		off += read
+		s -= read
+		e -= read
+		if left == 0 {
+			i++
+			off = 0
+		}
+	}
+	if s > 0 {
+		panic("pio: VecSlice start out of range")
+	}
+
+	for e != 0 && i < len(in) {
+		left := len(in[i])-off
+		read := left
+		if e > 0 && e < read {
+			read = e
+		}
+		out[n] = in[i][off:off+read]
+		n++
+		left -= read
+		e -= read
+		off += read
+		if left == 0 {
+			i++
+			off = 0
+		}
+	}
+	if e > 0 {
+		panic("pio: VecSlice end out of range")
+	}
+
+	return
+}
+
+func VecSlice(in [][]byte, s int, e int) (out [][]byte) {
+	out = make([][]byte, len(in))
+	n := VecSliceTo(in, out, s, e)
+	out = out[:n]
+	return
+}
+

utils/bits/pio/vec_test.go

@@ -0,0 +1,22 @@
+
+package pio
+
+import (
+	"fmt"
+)
+
+func ExampleVec() {
+	vec := [][]byte{[]byte{1,2,3}, []byte{4,5,6,7,8,9}, []byte{10,11,12,13}}
+	println(VecLen(vec))
+
+	vec = VecSlice(vec, 1, -1)
+	fmt.Println(vec)
+
+	vec = VecSlice(vec, 2, -1)
+	fmt.Println(vec)
+
+	vec = VecSlice(vec, 8, 8)
+	fmt.Println(vec)
+
+	// Output:
+}

utils/bits/pio/writer.go

@@ -0,0 +1,89 @@
+
+package pio
+
+func PutU8(b []byte, v uint8) {
+	b[0] = v
+}
+
+func PutI16BE(b []byte, v int16) {
+	b[0] = byte(v>>8)
+	b[1] = byte(v)
+}
+
+func PutU16BE(b []byte, v uint16) {
+	b[0] = byte(v>>8)
+	b[1] = byte(v)
+}
+
+func PutI24BE(b []byte, v int32) {
+	b[0] = byte(v>>16)
+	b[1] = byte(v>>8)
+	b[2] = byte(v)
+}
+
+func PutU24BE(b []byte, v uint32) {
+	b[0] = byte(v>>16)
+	b[1] = byte(v>>8)
+	b[2] = byte(v)
+}
+
+func PutI32BE(b []byte, v int32) {
+	b[0] = byte(v>>24)
+	b[1] = byte(v>>16)
+	b[2] = byte(v>>8)
+	b[3] = byte(v)
+}
+
+func PutU32BE(b []byte, v uint32) {
+	b[0] = byte(v>>24)
+	b[1] = byte(v>>16)
+	b[2] = byte(v>>8)
+	b[3] = byte(v)
+}
+
+func PutU32LE(b []byte, v uint32) {
+	b[3] = byte(v>>24)
+	b[2] = byte(v>>16)
+	b[1] = byte(v>>8)
+	b[0] = byte(v)
+}
+
+func PutU40BE(b []byte, v uint64) {
+	b[0] = byte(v>>32)
+	b[1] = byte(v>>24)
+	b[2] = byte(v>>16)
+	b[3] = byte(v>>8)
+	b[4] = byte(v)
+}
+
+func PutU48BE(b []byte, v uint64) {
+	b[0] = byte(v>>40)
+	b[1] = byte(v>>32)
+	b[2] = byte(v>>24)
+	b[3] = byte(v>>16)
+	b[4] = byte(v>>8)
+	b[5] = byte(v)
+}
+
+func PutU64BE(b []byte, v uint64) {
+	b[0] = byte(v>>56)
+	b[1] = byte(v>>48)
+	b[2] = byte(v>>40)
+	b[3] = byte(v>>32)
+	b[4] = byte(v>>24)
+	b[5] = byte(v>>16)
+	b[6] = byte(v>>8)
+	b[7] = byte(v)
+}
+
+func PutI64BE(b []byte, v int64) {
+	b[0] = byte(v>>56)
+	b[1] = byte(v>>48)
+	b[2] = byte(v>>40)
+	b[3] = byte(v>>32)
+	b[4] = byte(v>>24)
+	b[5] = byte(v>>16)
+	b[6] = byte(v>>8)
+	b[7] = byte(v)
+}
+