opus work in progress

av/av.go

@@ -123,6 +123,7 @@ var (
 	SPEEX      = MakeAudioCodecType(avCodecTypeMagic + 4)
 	NELLYMOSER = MakeAudioCodecType(avCodecTypeMagic + 5)
 	PCM        = MakeAudioCodecType(avCodecTypeMagic + 6)
+	OPUS       = MakeAudioCodecType(avCodecTypeMagic + 7)
 )
 
 const codecTypeAudioBit = 0x1
@@ -144,6 +145,8 @@ func (self CodecType) String() string {
 		return "NELLYMOSER"
 	case PCM:
 		return "PCM"
+	case OPUS:
+		return "OPUS"
 	}
 	return ""
 }

codec/codec.go

@@ -7,6 +7,32 @@ import (
 	"github.com/deepch/vdk/codec/fake"
 )
 
+type OpusCodecData struct {
+	typ            av.CodecType
+	SampleRate_    int
+	ChannelLayout_ av.ChannelLayout
+}
+
+func (self OpusCodecData) Type() av.CodecType {
+	return self.typ
+}
+
+func (self OpusCodecData) SampleRate() int {
+	return self.SampleRate_
+}
+
+func (self OpusCodecData) ChannelLayout() av.ChannelLayout {
+	return self.ChannelLayout_
+}
+
+func (self OpusCodecData) PacketDuration(data []byte) (time.Duration, error) {
+	return time.Duration(1000) * time.Second / time.Duration(self.SampleRate_), nil
+}
+
+func (self OpusCodecData) SampleFormat() av.SampleFormat {
+	return av.FLT
+}
+
 type PCMUCodecData struct {
 	typ av.CodecType
 }
@@ -48,15 +74,19 @@ func NewPCMAlawCodecData() av.AudioCodecData {
 		typ: av.PCM_ALAW,
 	}
 }
+func NewOpusCodecData(sr int, cc av.ChannelLayout) av.AudioCodecData {
+	return OpusCodecData{
+		typ:            av.OPUS,
+		SampleRate_:    sr,
+		ChannelLayout_: cc,
+	}
+}
 
 type SpeexCodecData struct {
 	fake.CodecData
 }
 
 func (self SpeexCodecData) PacketDuration(data []byte) (time.Duration, error) {
-	// libavcodec/libspeexdec.c
-	// samples = samplerate/50
-	// duration = 0.02s
 	return time.Millisecond * 20, nil
 }
 

format/rtsp/sdp/parser.go

@@ -20,6 +20,7 @@ type Media struct {
 	TimeScale          int
 	Control            string
 	Rtpmap             int
+	ChannelCount       int
 	Config             []byte
 	SpropParameterSets [][]byte
 	PayloadType        int
@@ -83,6 +84,13 @@ func Parse(content string) (sess Session, medias []Media) {
 								media.Type = av.AAC
 							case "L16":
 								media.Type = av.PCM
+							case "OPUS":
+								media.Type = av.OPUS
+								if len(keyval) > 2 {
+									if i, err := strconv.Atoi(keyval[2]); err == nil {
+										media.ChannelCount = i
+									}
+								}
 							case "H264":
 								media.Type = av.H264
 							}

format/rtspv2/client.go

@@ -75,7 +75,8 @@ type RTSPClient struct {
 	sps                 []byte
 	pps                 []byte
 	CodecData           []av.CodecData
-	PCMTime             int64
+	PCMTime             time.Duration
+	AudioTimeScale      int64
 }
 
 type RTSPClientOptions struct {
@@ -94,11 +95,12 @@ func Dial(options RTSPClientOptions) (*RTSPClient, error) {
 		OutgoingProxyQueue:  make(chan *[]byte, 3000),
 		OutgoingPacketQueue: make(chan *av.Packet, 3000),
 		BufferRtpPacket:     bytes.NewBuffer([]byte{}),
-		videoID:             0,
-		audioID:             2,
-		videoIDX:            0,
-		audioIDX:            1,
+		videoID:             -1,
+		audioID:             -2,
+		videoIDX:            -1,
+		audioIDX:            -2,
 		options:             options,
+		AudioTimeScale:      8000,
 	}
 	client.headers["User-Agent"] = "Lavf58.20.100"
 	err := client.parseURL(html.UnescapeString(client.options.URL))
@@ -154,10 +156,21 @@ func Dial(options RTSPClientOptions) (*RTSPClient, error) {
 				if err == nil {
 					client.Println("Audio AAC bad config")
 				}
+			case av.OPUS:
+				var cl av.ChannelLayout
+				switch i2.ChannelCount {
+				case 1:
+					cl = av.CH_MONO
+				case 2:
+					cl = av.CH_STEREO
+				default:
+					cl = av.CH_MONO
+				}
+				CodecData = codec.NewOpusCodecData(i2.TimeScale, cl)
 			case av.PCM_MULAW:
 				CodecData = codec.NewPCMMulawCodecData()
 			case av.PCM_ALAW:
-				CodecData = codec.NewPCMMulawCodecData()
+				CodecData = codec.NewPCMAlawCodecData()
 			case av.PCM:
 				CodecData = codec.NewPCMCodecData()
 			default:
@@ -166,6 +179,9 @@ func Dial(options RTSPClientOptions) (*RTSPClient, error) {
 			if CodecData != nil {
 				client.CodecData = append(client.CodecData, CodecData)
 				client.audioIDX = int8(len(client.CodecData) - 1)
+				if i2.TimeScale != 0 {
+					client.AudioTimeScale = int64(i2.TimeScale)
+				}
 			}
 		}
 		ch += 2
@@ -556,14 +572,21 @@ func (client *RTSPClient) RTPDemuxer(payloadRAW *[]byte) ([]*av.Packet, bool) {
 		nalRaw, _ := h264parser.SplitNALUs(content[offset:end])
 		var retmap []*av.Packet
 		for _, nal := range nalRaw {
-			//client.PCMTime += int64(float32(1000) / (float32(8000) / float32(len(nal))))
+			//basic
+			//time.Duration(float32(timestamp)/float32(float32(client.AudioTimeScale)/float32(1000))) * time.Millisecond
+			//pcm
+			//client.PCMTime += time.Duration(len(nal)) * time.Second / time.Duration(client.AudioTimeScale)
+			//opus
+			//client.PCMTime := time.Duration((sampleCount/48000)*1000) * time.Millisecond
+			//Need Add Opus And AAC
 			retmap = append(retmap, &av.Packet{
 				Data:            append(binSize(len(nal)), nal...),
 				CompositionTime: time.Duration(1) * time.Millisecond,
 				Idx:             client.audioIDX,
 				IsKeyFrame:      false,
-				Time:            time.Duration(timestamp/8) * time.Millisecond,
+				Time:            time.Duration(float32(timestamp)/float32(float32(client.AudioTimeScale)/float32(1000))) * time.Millisecond,
 			})
+			//log.Println("===>", time.Duration(float32(timestamp)/float32(float32(client.AudioTimeScale)/float32(1000)))*time.Millisecond)
 		}
 		if len(retmap) > 0 {
 			return retmap, true

format/webrtcv3/adapter.go

@@ -14,23 +14,6 @@ import (
 	"github.com/pion/webrtc/v3/pkg/media"
 )
 
-const (
-	// MimeTypeH264 H264 MIME type.
-	MimeTypeH264 = "video/h264"
-	// MimeTypeOpus Opus MIME type
-	MimeTypeOpus = "audio/opus"
-	// MimeTypeVP8 VP8 MIME type
-	MimeTypeVP8 = "video/vp8"
-	// MimeTypeVP9 VP9 MIME type
-	MimeTypeVP9 = "video/vp9"
-	// MimeTypeG722 G722 MIME type
-	MimeTypeG722 = "audio/G722"
-	// MimeTypePCMU PCMU MIME type
-	MimeTypePCMU = "audio/PCMU"
-	// MimeTypePCMA PCMA MIME type
-	MimeTypePCMA = "audio/PCMA"
-)
-
 var (
 	ErrorNotFound          = errors.New("WebRTC Stream Not Found")
 	ErrorCodecNotSupported = errors.New("WebRTC Codec Not Supported")
@@ -99,18 +82,22 @@ func (element *Muxer) WriteHeader(streams []av.CodecData, sdp64 string) (string,
 				}
 			}
 		} else if i2.Type().IsAudio() {
-			AudioCodecString := MimeTypePCMU
+			AudioCodecString := webrtc.MimeTypePCMA
 			switch i2.Type() {
 			case av.PCM_ALAW:
-				AudioCodecString = MimeTypePCMA
+				AudioCodecString = webrtc.MimeTypePCMA
 			case av.PCM_MULAW:
-				AudioCodecString = MimeTypePCMU
+				AudioCodecString = webrtc.MimeTypePCMU
+			case av.OPUS:
+				AudioCodecString = webrtc.MimeTypeOpus
 			default:
 				log.Println(ErrorIgnoreAudioTrack)
 				continue
 			}
 			track, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{
 				MimeType:  AudioCodecString,
+				Channels:  uint16(i2.(av.AudioCodecData).ChannelLayout().Count()),
+				ClockRate: uint32(i2.(av.AudioCodecData).SampleRate()),
 			}, "pion-rtsp-audio", "pion-rtsp-audio")
 			if err != nil {
 				return "", err
@@ -162,6 +149,7 @@ func (element *Muxer) WriteHeader(streams []av.CodecData, sdp64 string) (string,
 }
 
 func (element *Muxer) WritePacket(pkt av.Packet) (err error) {
+	//log.Println("WritePacket", pkt.Time, element.stop, webrtc.ICEConnectionStateConnected, pkt.Idx, element.streams[pkt.Idx])
 	var WritePacketSuccess bool
 	defer func() {
 		if !WritePacketSuccess {
@@ -189,6 +177,7 @@ func (element *Muxer) WritePacket(pkt av.Packet) (err error) {
 			}
 		case av.PCM_MULAW:
 		case av.PCM_ALAW:
+		case av.OPUS:
 		default:
 			return ErrorCodecNotSupported
 		}