From 6a92c34cd460beb4be5a39c95b455d0699382f6b Mon Sep 17 00:00:00 2001 From: Andrey Semochkin Date: Sat, 9 Jan 2021 07:37:19 +0300 Subject: [PATCH] opus work in progress --- av/av.go | 3 +++ codec/codec.go | 36 ++++++++++++++++++++++++++++++++--- format/rtsp/sdp/parser.go | 8 ++++++++ format/rtspv2/client.go | 39 ++++++++++++++++++++++++++++++-------- format/webrtcv3/adapter.go | 31 ++++++++++-------------------- 5 files changed, 85 insertions(+), 32 deletions(-) diff --git a/av/av.go b/av/av.go index 3589872..44ecdb4 100644 --- a/av/av.go +++ b/av/av.go @@ -123,6 +123,7 @@ var ( SPEEX = MakeAudioCodecType(avCodecTypeMagic + 4) NELLYMOSER = MakeAudioCodecType(avCodecTypeMagic + 5) PCM = MakeAudioCodecType(avCodecTypeMagic + 6) + OPUS = MakeAudioCodecType(avCodecTypeMagic + 7) ) const codecTypeAudioBit = 0x1 @@ -144,6 +145,8 @@ func (self CodecType) String() string { return "NELLYMOSER" case PCM: return "PCM" + case OPUS: + return "OPUS" } return "" } diff --git a/codec/codec.go b/codec/codec.go index 70b1eb7..035f0c0 100644 --- a/codec/codec.go +++ b/codec/codec.go @@ -7,6 +7,32 @@ import ( "github.com/deepch/vdk/codec/fake" ) +type OpusCodecData struct { + typ av.CodecType + SampleRate_ int + ChannelLayout_ av.ChannelLayout +} + +func (self OpusCodecData) Type() av.CodecType { + return self.typ +} + +func (self OpusCodecData) SampleRate() int { + return self.SampleRate_ +} + +func (self OpusCodecData) ChannelLayout() av.ChannelLayout { + return self.ChannelLayout_ +} + +func (self OpusCodecData) PacketDuration(data []byte) (time.Duration, error) { + return time.Duration(1000) * time.Second / time.Duration(self.SampleRate_), nil +} + +func (self OpusCodecData) SampleFormat() av.SampleFormat { + return av.FLT +} + type PCMUCodecData struct { typ av.CodecType } @@ -48,15 +74,19 @@ func NewPCMAlawCodecData() av.AudioCodecData { typ: av.PCM_ALAW, } } +func NewOpusCodecData(sr int, cc av.ChannelLayout) av.AudioCodecData { + return OpusCodecData{ + typ: av.OPUS, + SampleRate_: sr, + ChannelLayout_: cc, + } +} type SpeexCodecData struct { fake.CodecData } func (self SpeexCodecData) PacketDuration(data []byte) (time.Duration, error) { - // libavcodec/libspeexdec.c - // samples = samplerate/50 - // duration = 0.02s return time.Millisecond * 20, nil } diff --git a/format/rtsp/sdp/parser.go b/format/rtsp/sdp/parser.go index 90441f0..40ea353 100644 --- a/format/rtsp/sdp/parser.go +++ b/format/rtsp/sdp/parser.go @@ -20,6 +20,7 @@ type Media struct { TimeScale int Control string Rtpmap int + ChannelCount int Config []byte SpropParameterSets [][]byte PayloadType int @@ -83,6 +84,13 @@ func Parse(content string) (sess Session, medias []Media) { media.Type = av.AAC case "L16": media.Type = av.PCM + case "OPUS": + media.Type = av.OPUS + if len(keyval) > 2 { + if i, err := strconv.Atoi(keyval[2]); err == nil { + media.ChannelCount = i + } + } case "H264": media.Type = av.H264 } diff --git a/format/rtspv2/client.go b/format/rtspv2/client.go index 28a0b08..37af914 100644 --- a/format/rtspv2/client.go +++ b/format/rtspv2/client.go @@ -75,7 +75,8 @@ type RTSPClient struct { sps []byte pps []byte CodecData []av.CodecData - PCMTime int64 + PCMTime time.Duration + AudioTimeScale int64 } type RTSPClientOptions struct { @@ -94,11 +95,12 @@ func Dial(options RTSPClientOptions) (*RTSPClient, error) { OutgoingProxyQueue: make(chan *[]byte, 3000), OutgoingPacketQueue: make(chan *av.Packet, 3000), BufferRtpPacket: bytes.NewBuffer([]byte{}), - videoID: 0, - audioID: 2, - videoIDX: 0, - audioIDX: 1, + videoID: -1, + audioID: -2, + videoIDX: -1, + audioIDX: -2, options: options, + AudioTimeScale: 8000, } client.headers["User-Agent"] = "Lavf58.20.100" err := client.parseURL(html.UnescapeString(client.options.URL)) @@ -154,10 +156,21 @@ func Dial(options RTSPClientOptions) (*RTSPClient, error) { if err == nil { client.Println("Audio AAC bad config") } + case av.OPUS: + var cl av.ChannelLayout + switch i2.ChannelCount { + case 1: + cl = av.CH_MONO + case 2: + cl = av.CH_STEREO + default: + cl = av.CH_MONO + } + CodecData = codec.NewOpusCodecData(i2.TimeScale, cl) case av.PCM_MULAW: CodecData = codec.NewPCMMulawCodecData() case av.PCM_ALAW: - CodecData = codec.NewPCMMulawCodecData() + CodecData = codec.NewPCMAlawCodecData() case av.PCM: CodecData = codec.NewPCMCodecData() default: @@ -166,6 +179,9 @@ func Dial(options RTSPClientOptions) (*RTSPClient, error) { if CodecData != nil { client.CodecData = append(client.CodecData, CodecData) client.audioIDX = int8(len(client.CodecData) - 1) + if i2.TimeScale != 0 { + client.AudioTimeScale = int64(i2.TimeScale) + } } } ch += 2 @@ -556,14 +572,21 @@ func (client *RTSPClient) RTPDemuxer(payloadRAW *[]byte) ([]*av.Packet, bool) { nalRaw, _ := h264parser.SplitNALUs(content[offset:end]) var retmap []*av.Packet for _, nal := range nalRaw { - //client.PCMTime += int64(float32(1000) / (float32(8000) / float32(len(nal)))) + //basic + //time.Duration(float32(timestamp)/float32(float32(client.AudioTimeScale)/float32(1000))) * time.Millisecond + //pcm + //client.PCMTime += time.Duration(len(nal)) * time.Second / time.Duration(client.AudioTimeScale) + //opus + //client.PCMTime := time.Duration((sampleCount/48000)*1000) * time.Millisecond + //Need Add Opus And AAC retmap = append(retmap, &av.Packet{ Data: append(binSize(len(nal)), nal...), CompositionTime: time.Duration(1) * time.Millisecond, Idx: client.audioIDX, IsKeyFrame: false, - Time: time.Duration(timestamp/8) * time.Millisecond, + Time: time.Duration(float32(timestamp)/float32(float32(client.AudioTimeScale)/float32(1000))) * time.Millisecond, }) + //log.Println("===>", time.Duration(float32(timestamp)/float32(float32(client.AudioTimeScale)/float32(1000)))*time.Millisecond) } if len(retmap) > 0 { return retmap, true diff --git a/format/webrtcv3/adapter.go b/format/webrtcv3/adapter.go index b89f76d..a372634 100644 --- a/format/webrtcv3/adapter.go +++ b/format/webrtcv3/adapter.go @@ -14,23 +14,6 @@ import ( "github.com/pion/webrtc/v3/pkg/media" ) -const ( - // MimeTypeH264 H264 MIME type. - MimeTypeH264 = "video/h264" - // MimeTypeOpus Opus MIME type - MimeTypeOpus = "audio/opus" - // MimeTypeVP8 VP8 MIME type - MimeTypeVP8 = "video/vp8" - // MimeTypeVP9 VP9 MIME type - MimeTypeVP9 = "video/vp9" - // MimeTypeG722 G722 MIME type - MimeTypeG722 = "audio/G722" - // MimeTypePCMU PCMU MIME type - MimeTypePCMU = "audio/PCMU" - // MimeTypePCMA PCMA MIME type - MimeTypePCMA = "audio/PCMA" -) - var ( ErrorNotFound = errors.New("WebRTC Stream Not Found") ErrorCodecNotSupported = errors.New("WebRTC Codec Not Supported") @@ -99,18 +82,22 @@ func (element *Muxer) WriteHeader(streams []av.CodecData, sdp64 string) (string, } } } else if i2.Type().IsAudio() { - AudioCodecString := MimeTypePCMU + AudioCodecString := webrtc.MimeTypePCMA switch i2.Type() { case av.PCM_ALAW: - AudioCodecString = MimeTypePCMA + AudioCodecString = webrtc.MimeTypePCMA case av.PCM_MULAW: - AudioCodecString = MimeTypePCMU + AudioCodecString = webrtc.MimeTypePCMU + case av.OPUS: + AudioCodecString = webrtc.MimeTypeOpus default: log.Println(ErrorIgnoreAudioTrack) continue } track, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{ - MimeType: AudioCodecString, + MimeType: AudioCodecString, + Channels: uint16(i2.(av.AudioCodecData).ChannelLayout().Count()), + ClockRate: uint32(i2.(av.AudioCodecData).SampleRate()), }, "pion-rtsp-audio", "pion-rtsp-audio") if err != nil { return "", err @@ -162,6 +149,7 @@ func (element *Muxer) WriteHeader(streams []av.CodecData, sdp64 string) (string, } func (element *Muxer) WritePacket(pkt av.Packet) (err error) { + //log.Println("WritePacket", pkt.Time, element.stop, webrtc.ICEConnectionStateConnected, pkt.Idx, element.streams[pkt.Idx]) var WritePacketSuccess bool defer func() { if !WritePacketSuccess { @@ -189,6 +177,7 @@ func (element *Muxer) WritePacket(pkt av.Packet) (err error) { } case av.PCM_MULAW: case av.PCM_ALAW: + case av.OPUS: default: return ErrorCodecNotSupported }