diff --git a/BroadcastUploadExtension/Common.xcconfig b/BroadcastUploadExtension/Common.xcconfig new file mode 100644 index 000000000..7d3233774 --- /dev/null +++ b/BroadcastUploadExtension/Common.xcconfig @@ -0,0 +1,33 @@ +// +// Copyright 2020 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Configuration settings file format documentation can be found at: +// https://help.apple.com/xcode/#/dev745c5c974 + +#include "Config/AppIdentifiers.xcconfig" +#include "Config/AppVersion.xcconfig" + +PRODUCT_NAME = BroadcastUploadExtension +PRODUCT_BUNDLE_IDENTIFIER = $(BASE_BUNDLE_IDENTIFIER).broadcastUploadExtension + +INFOPLIST_FILE = BroadcastUploadExtension/SupportingFiles/Info.plist +ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon + +APPLICATION_EXTENSION_API_ONLY = YES +SKIP_INSTALL = YES +LD_RUNPATH_SEARCH_PATHS = $(inherited) @executable_path/Frameworks @executable_path/../../Frameworks + +SWIFT_OBJC_INTERFACE_HEADER_NAME = GeneratedInterface-Swift.h diff --git a/BroadcastUploadExtension/Debug.xcconfig b/BroadcastUploadExtension/Debug.xcconfig new file mode 100644 index 000000000..61e55f3e6 --- /dev/null +++ b/BroadcastUploadExtension/Debug.xcconfig @@ -0,0 +1,20 @@ +// +// Copyright 2020 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Configuration settings file format documentation can be found at: +// https://help.apple.com/xcode/#/dev745c5c974 + +#include "Common.xcconfig" diff --git a/BroadcastUploadExtension/Release.xcconfig b/BroadcastUploadExtension/Release.xcconfig new file mode 100644 index 000000000..45d0bb011 --- /dev/null +++ b/BroadcastUploadExtension/Release.xcconfig @@ -0,0 +1,26 @@ +// +// Copyright 2020 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Configuration settings file format documentation can be found at: +// https://help.apple.com/xcode/#/dev745c5c974 + +#include "Common.xcconfig" + +// Add the provisioning profiles when they are made +// PROVISIONING_PROFILE = $(SHARE_EXTENSION_PROVISIONING_PROFILE) +// PROVISIONING_PROFILE_SPECIFIER = $(SHARE_EXTENSION_PROVISIONING_PROFILE_SPECIFIER) + +COPY_PHASE_STRIP = NO diff --git a/BroadcastUploadExtension/Sources/Atomic.swift b/BroadcastUploadExtension/Sources/Atomic.swift new file mode 100644 index 000000000..248410f65 --- /dev/null +++ b/BroadcastUploadExtension/Sources/Atomic.swift @@ -0,0 +1,37 @@ +// +// Atomic.swift +// Broadcast Extension +// +// Created by Maksym Shcheglov. +// https://www.onswiftwings.com/posts/atomic-property-wrapper/ +// + +import Foundation + +@propertyWrapper +struct Atomic { + + private var value: Value + private let lock = NSLock() + + init(wrappedValue value: Value) { + self.value = value + } + + var wrappedValue: Value { + get { load() } + set { store(newValue: newValue) } + } + + func load() -> Value { + lock.lock() + defer { lock.unlock() } + return value + } + + mutating func store(newValue: Value) { + lock.lock() + defer { lock.unlock() } + value = newValue + } +} diff --git a/BroadcastUploadExtension/Sources/DarwinNotificationCenter.swift b/BroadcastUploadExtension/Sources/DarwinNotificationCenter.swift new file mode 100644 index 000000000..3501486df --- /dev/null +++ b/BroadcastUploadExtension/Sources/DarwinNotificationCenter.swift @@ -0,0 +1,29 @@ +// +// DarwinNotificationCenter.swift +// Broadcast Extension +// +// Created by Alex-Dan Bumbu on 23/03/2021. +// Copyright © 2021 8x8, Inc. All rights reserved. +// + +import Foundation + +enum DarwinNotification: String { + case broadcastStarted = "iOS_BroadcastStarted" + case broadcastStopped = "iOS_BroadcastStopped" +} + +class DarwinNotificationCenter { + + static let shared = DarwinNotificationCenter() + + private let notificationCenter: CFNotificationCenter + + init() { + notificationCenter = CFNotificationCenterGetDarwinNotifyCenter() + } + + func postNotification(_ name: DarwinNotification) { + CFNotificationCenterPostNotification(notificationCenter, CFNotificationName(rawValue: name.rawValue as CFString), nil, nil, true) + } +} diff --git a/BroadcastUploadExtension/Sources/SampleHandler.swift b/BroadcastUploadExtension/Sources/SampleHandler.swift new file mode 100644 index 000000000..ae2f76555 --- /dev/null +++ b/BroadcastUploadExtension/Sources/SampleHandler.swift @@ -0,0 +1,104 @@ +// +// SampleHandler.swift +// Broadcast Extension +// +// Created by Alex-Dan Bumbu on 04.06.2021. +// + +import ReplayKit + +private enum Constants { + // the App Group ID value that the app and the broadcast extension targets are setup with. It differs for each app. + static let appGroupIdentifier = "group.com.jitsi.example-screensharing.appgroup" +} + +class SampleHandler: RPBroadcastSampleHandler { + + private var clientConnection: SocketConnection? + private var uploader: SampleUploader? + + private var frameCount: Int = 0 + + var socketFilePath: String { + let sharedContainer = FileManager.default.containerURL(forSecurityApplicationGroupIdentifier: Constants.appGroupIdentifier) + return sharedContainer?.appendingPathComponent("rtc_SSFD").path ?? "" + } + + override init() { + super.init() + if let connection = SocketConnection(filePath: socketFilePath) { + clientConnection = connection + setupConnection() + + uploader = SampleUploader(connection: connection) + } + } + + override func broadcastStarted(withSetupInfo setupInfo: [String: NSObject]?) { + // User has requested to start the broadcast. Setup info from the UI extension can be supplied but optional. + frameCount = 0 + + DarwinNotificationCenter.shared.postNotification(.broadcastStarted) + openConnection() + } + + override func broadcastPaused() { + // User has requested to pause the broadcast. Samples will stop being delivered. + } + + override func broadcastResumed() { + // User has requested to resume the broadcast. Samples delivery will resume. + } + + override func broadcastFinished() { + // User has requested to finish the broadcast. + DarwinNotificationCenter.shared.postNotification(.broadcastStopped) + clientConnection?.close() + } + + override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) { + switch sampleBufferType { + case RPSampleBufferType.video: + // very simple mechanism for adjusting frame rate by using every third frame + frameCount += 1 + if frameCount % 3 == 0 { + uploader?.send(sample: sampleBuffer) + } + default: + break + } + } +} + +private extension SampleHandler { + + func setupConnection() { + clientConnection?.didClose = { [weak self] error in + print("client connection did close \(String(describing: error))") + + if let error = error { + self?.finishBroadcastWithError(error) + } else { + // the displayed failure message is more user friendly when using NSError instead of Error + let JMScreenSharingStopped = 10001 + let customError = NSError(domain: RPRecordingErrorDomain, code: JMScreenSharingStopped, userInfo: [NSLocalizedDescriptionKey: "Screen sharing stopped"]) + self?.finishBroadcastWithError(customError) + } + } + } + + func openConnection() { + let queue = DispatchQueue(label: "broadcast.connectTimer") + let timer = DispatchSource.makeTimerSource(queue: queue) + timer.schedule(deadline: .now(), repeating: .milliseconds(100), leeway: .milliseconds(500)) + timer.setEventHandler { [weak self] in + guard self?.clientConnection?.open() == true else { + return + } + + timer.cancel() + } + + timer.resume() + } +} diff --git a/BroadcastUploadExtension/Sources/SampleUploader.swift b/BroadcastUploadExtension/Sources/SampleUploader.swift new file mode 100644 index 000000000..084deaed0 --- /dev/null +++ b/BroadcastUploadExtension/Sources/SampleUploader.swift @@ -0,0 +1,146 @@ +// +// SampleUploader.swift +// Broadcast Extension +// +// Created by Alex-Dan Bumbu on 22/03/2021. +// Copyright © 2021 8x8, Inc. All rights reserved. +// + +import Foundation +import ReplayKit + +private enum Constants { + static let bufferMaxLength = 10240 +} + +class SampleUploader { + + private static var imageContext = CIContext(options: nil) + + @Atomic private var isReady = false + private var connection: SocketConnection + + private var dataToSend: Data? + private var byteIndex = 0 + + private let serialQueue: DispatchQueue + + init(connection: SocketConnection) { + self.connection = connection + self.serialQueue = DispatchQueue(label: "org.jitsi.meet.broadcast.sampleUploader") + + setupConnection() + } + + @discardableResult func send(sample buffer: CMSampleBuffer) -> Bool { + guard isReady else { + return false + } + + isReady = false + + dataToSend = prepare(sample: buffer) + byteIndex = 0 + + serialQueue.async { [weak self] in + self?.sendDataChunk() + } + + return true + } +} + +private extension SampleUploader { + + func setupConnection() { + connection.didOpen = { [weak self] in + self?.isReady = true + } + connection.streamHasSpaceAvailable = { [weak self] in + self?.serialQueue.async { + if let success = self?.sendDataChunk() { + self?.isReady = !success + } + } + } + } + + @discardableResult func sendDataChunk() -> Bool { + guard let dataToSend = dataToSend else { + return false + } + + var bytesLeft = dataToSend.count - byteIndex + var length = bytesLeft > Constants.bufferMaxLength ? Constants.bufferMaxLength : bytesLeft + + length = dataToSend[byteIndex..<(byteIndex + length)].withUnsafeBytes { + guard let ptr = $0.bindMemory(to: UInt8.self).baseAddress else { + return 0 + } + + return connection.writeToStream(buffer: ptr, maxLength: length) + } + + if length > 0 { + byteIndex += length + bytesLeft -= length + + if bytesLeft == 0 { + self.dataToSend = nil + byteIndex = 0 + } + } else { + print("writeBufferToStream failure") + } + + return true + } + + func prepare(sample buffer: CMSampleBuffer) -> Data? { + guard let imageBuffer = CMSampleBufferGetImageBuffer(buffer) else { + print("image buffer not available") + return nil + } + + CVPixelBufferLockBaseAddress(imageBuffer, .readOnly) + + let scaleFactor = 2.0 + let width = CVPixelBufferGetWidth(imageBuffer)/Int(scaleFactor) + let height = CVPixelBufferGetHeight(imageBuffer)/Int(scaleFactor) + let orientation = CMGetAttachment(buffer, key: RPVideoSampleOrientationKey as CFString, attachmentModeOut: nil)?.uintValue ?? 0 + + let scaleTransform = CGAffineTransform(scaleX: CGFloat(1.0/scaleFactor), y: CGFloat(1.0/scaleFactor)) + let bufferData = self.jpegData(from: imageBuffer, scale: scaleTransform) + + CVPixelBufferUnlockBaseAddress(imageBuffer, .readOnly) + + guard let messageData = bufferData else { + print("corrupted image buffer") + return nil + } + + let httpResponse = CFHTTPMessageCreateResponse(nil, 200, nil, kCFHTTPVersion1_1).takeRetainedValue() + CFHTTPMessageSetHeaderFieldValue(httpResponse, "Content-Length" as CFString, String(messageData.count) as CFString) + CFHTTPMessageSetHeaderFieldValue(httpResponse, "Buffer-Width" as CFString, String(width) as CFString) + CFHTTPMessageSetHeaderFieldValue(httpResponse, "Buffer-Height" as CFString, String(height) as CFString) + CFHTTPMessageSetHeaderFieldValue(httpResponse, "Buffer-Orientation" as CFString, String(orientation) as CFString) + + CFHTTPMessageSetBody(httpResponse, messageData as CFData) + + let serializedMessage = CFHTTPMessageCopySerializedMessage(httpResponse)?.takeRetainedValue() as Data? + + return serializedMessage + } + + func jpegData(from buffer: CVPixelBuffer, scale scaleTransform: CGAffineTransform) -> Data? { + let image = CIImage(cvPixelBuffer: buffer).transformed(by: scaleTransform) + + guard let colorSpace = image.colorSpace else { + return nil + } + + let options: [CIImageRepresentationOption: Float] = [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 1.0] + + return SampleUploader.imageContext.jpegRepresentation(of: image, colorSpace: colorSpace, options: options) + } +} diff --git a/BroadcastUploadExtension/Sources/SocketConnection.swift b/BroadcastUploadExtension/Sources/SocketConnection.swift new file mode 100644 index 000000000..4d178251c --- /dev/null +++ b/BroadcastUploadExtension/Sources/SocketConnection.swift @@ -0,0 +1,198 @@ +// +// SocketConnection.swift +// Broadcast Extension +// +// Created by Alex-Dan Bumbu on 22/03/2021. +// Copyright © 2021 Atlassian Inc. All rights reserved. +// + +import Foundation + +class SocketConnection: NSObject { + var didOpen: (() -> Void)? + var didClose: ((Error?) -> Void)? + var streamHasSpaceAvailable: (() -> Void)? + + private let filePath: String + private var socketHandle: Int32 = -1 + private var address: sockaddr_un? + + private var inputStream: InputStream? + private var outputStream: OutputStream? + + private var networkQueue: DispatchQueue? + private var shouldKeepRunning = false + + init?(filePath path: String) { + filePath = path + socketHandle = Darwin.socket(AF_UNIX, SOCK_STREAM, 0) + + guard socketHandle != -1 else { + print("failure: create socket") + return nil + } + } + + func open() -> Bool { + print("open socket connection") + + guard FileManager.default.fileExists(atPath: filePath) else { + print("failure: socket file missing") + return false + } + + guard setupAddress() == true else { + return false + } + + guard connectSocket() == true else { + return false + } + + setupStreams() + + inputStream?.open() + outputStream?.open() + + return true + } + + func close() { + unscheduleStreams() + + inputStream?.delegate = nil + outputStream?.delegate = nil + + inputStream?.close() + outputStream?.close() + + inputStream = nil + outputStream = nil + } + + func writeToStream(buffer: UnsafePointer, maxLength length: Int) -> Int { + outputStream?.write(buffer, maxLength: length) ?? 0 + } +} + +extension SocketConnection: StreamDelegate { + + func stream(_ aStream: Stream, handle eventCode: Stream.Event) { + switch eventCode { + case .openCompleted: + print("client stream open completed") + if aStream == outputStream { + didOpen?() + } + case .hasBytesAvailable: + if aStream == inputStream { + var buffer: UInt8 = 0 + let numberOfBytesRead = inputStream?.read(&buffer, maxLength: 1) + if numberOfBytesRead == 0 && aStream.streamStatus == .atEnd { + print("server socket closed") + close() + notifyDidClose(error: nil) + } + } + case .hasSpaceAvailable: + if aStream == outputStream { + streamHasSpaceAvailable?() + } + case .errorOccurred: + print("client stream error occured: \(String(describing: aStream.streamError))") + close() + notifyDidClose(error: aStream.streamError) + + default: + break + } + } +} + +private extension SocketConnection { + + func setupAddress() -> Bool { + var addr = sockaddr_un() + guard filePath.count < MemoryLayout.size(ofValue: addr.sun_path) else { + print("failure: fd path is too long") + return false + } + + _ = withUnsafeMutablePointer(to: &addr.sun_path.0) { ptr in + filePath.withCString { + strncpy(ptr, $0, filePath.count) + } + } + + address = addr + return true + } + + func connectSocket() -> Bool { + guard var addr = address else { + return false + } + + let status = withUnsafePointer(to: &addr) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { + Darwin.connect(socketHandle, $0, socklen_t(MemoryLayout.size)) + } + } + + guard status == noErr else { + print("failure: \(status)") + return false + } + + return true + } + + func setupStreams() { + var readStream: Unmanaged? + var writeStream: Unmanaged? + + CFStreamCreatePairWithSocket(kCFAllocatorDefault, socketHandle, &readStream, &writeStream) + + inputStream = readStream?.takeRetainedValue() + inputStream?.delegate = self + inputStream?.setProperty(kCFBooleanTrue, forKey: Stream.PropertyKey(kCFStreamPropertyShouldCloseNativeSocket as String)) + + outputStream = writeStream?.takeRetainedValue() + outputStream?.delegate = self + outputStream?.setProperty(kCFBooleanTrue, forKey: Stream.PropertyKey(kCFStreamPropertyShouldCloseNativeSocket as String)) + + scheduleStreams() + } + + func scheduleStreams() { + shouldKeepRunning = true + + networkQueue = DispatchQueue.global(qos: .userInitiated) + networkQueue?.async { [weak self] in + self?.inputStream?.schedule(in: .current, forMode: .common) + self?.outputStream?.schedule(in: .current, forMode: .common) + RunLoop.current.run() + + var isRunning = false + + repeat { + isRunning = self?.shouldKeepRunning ?? false && RunLoop.current.run(mode: .default, before: .distantFuture) + } while (isRunning) + } + } + + func unscheduleStreams() { + networkQueue?.sync { [weak self] in + self?.inputStream?.remove(from: .current, forMode: .common) + self?.outputStream?.remove(from: .current, forMode: .common) + } + + shouldKeepRunning = false + } + + func notifyDidClose(error: Error?) { + if didClose != nil { + didClose?(error) + } + } +} diff --git a/BroadcastUploadExtension/SupportingFiles/Info.plist b/BroadcastUploadExtension/SupportingFiles/Info.plist new file mode 100644 index 000000000..34e3187b8 --- /dev/null +++ b/BroadcastUploadExtension/SupportingFiles/Info.plist @@ -0,0 +1,33 @@ + + + + + CFBundleVersion + $(CURRENT_PROJECT_VERSION) + CFBundleShortVersionString + $(MARKETING_VERSION) + CFBundlePackageType + XPC! + CFBundleName + $(PRODUCT_NAME) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleDisplayName + $(BUNDLE_DISPLAY_NAME) + CFBundleDevelopmentRegion + en + NSExtension + + NSExtensionPointIdentifier + com.apple.broadcast-services-upload + NSExtensionPrincipalClass + $(PRODUCT_MODULE_NAME).SampleHandler + RPBroadcastProcessMode + RPBroadcastProcessModeSampleBuffer + + + diff --git a/BroadcastUploadExtension/target.yml b/BroadcastUploadExtension/target.yml new file mode 100644 index 000000000..1af924d8d --- /dev/null +++ b/BroadcastUploadExtension/target.yml @@ -0,0 +1,39 @@ +name: BroadcastUploadExtension + +schemes: + BroadcastUploadExtension: + analyze: + config: Debug + archive: + config: Release + build: + targets: + BroadcastUploadExtension: + - running + - testing + - profiling + - analyzing + - archiving + profile: + config: Release + run: + askForAppToLaunch: true + config: Debug + debugEnabled: false + disableMainThreadChecker: true + launchAutomaticallySubstyle: 2 + test: + config: Debug + disableMainThreadChecker: true + +targets: + BroadcastUploadExtension: + platform: iOS + type: app-extension + + configFiles: + Debug: Debug.xcconfig + Release: Release.xcconfig + + sources: + - path: . diff --git a/project.yml b/project.yml index 3922de651..449ee955c 100644 --- a/project.yml +++ b/project.yml @@ -32,6 +32,7 @@ include: - path: RiotShareExtension/target.yml - path: SiriIntents/target.yml - path: RiotNSE/target.yml + - path: BroadcastUploadExtension/target.yml - path: DesignKit/target.yml - path: RiotSwiftUI/target.yml - path: RiotSwiftUI/targetUnitTests.yml