OpenShot Library | libopenshot  0.2.5
Frame.h
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Header file for Frame class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC
12  * <http://www.openshotstudios.com/>. This file is part of
13  * OpenShot Library (libopenshot), an open-source project dedicated to
14  * delivering high quality video editing and animation solutions to the
15  * world. For more information visit <http://www.openshot.org/>.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #ifndef OPENSHOT_FRAME_H
32 #define OPENSHOT_FRAME_H
33 
34 #include <iomanip>
35 #include <sstream>
36 #include <queue>
37 #include <QtWidgets/QApplication>
38 #include <QtGui/QImage>
39 #include <QtGui/QColor>
40 #include <QtGui/QBitmap>
41 #include <QtCore/QString>
42 #include <QtCore/QVector>
43 #include <QtGui/QPainter>
44 #include <QtWidgets/QHBoxLayout>
45 #include <QtWidgets/QWidget>
46 #include <QtWidgets/QLabel>
47 #include <memory>
48 #include <unistd.h>
49 #include "ZmqLogger.h"
50 #include "ChannelLayouts.h"
51 #include "AudioBufferSource.h"
52 #include "AudioResampler.h"
53 #include "Fraction.h"
54 #include "JuceHeader.h"
55 #ifdef USE_IMAGEMAGICK
56  #include "MagickUtilities.h"
57 #endif
58 
59 namespace openshot
60 {
61  /**
62  * @brief This class represents a single frame of video (i.e. image & audio data)
63  *
64  * FileReaders (such as FFmpegReader) use instances of this class to store the individual frames of video,
65  * which include both the image data (i.e. pixels) and audio samples. An openshot::Frame also has many debug
66  * methods, such as the ability to display the image (using X11), play the audio samples (using JUCE), or
67  * display the audio waveform as an image.
68  *
69  * FileWriters (such as FFmpegWriter) use instances of this class to create new video files, image files, or
70  * video streams. So, think of these openshot::Frame instances as the smallest unit of work in a video
71  * editor.
72  *
73  * There are many ways to create an instance of an openshot::Frame:
74  * @code
75  *
76  * // Most basic: a blank frame (300x200 blank image, 48kHz audio silence)
77  * Frame();
78  *
79  * // Image only settings (48kHz audio silence)
80  * Frame(1, // Frame number
81  * 720, // Width of image
82  * 480, // Height of image
83  * "#000000" // HTML color code of background color
84  * );
85  *
86  * // Audio only (300x200 blank image)
87  * Frame(number, // Frame number
88  * 44100, // Sample rate of audio stream
89  * 2 // Number of audio channels
90  * );
91  *
92  * // Image and Audio settings (user defines all key settings)
93  * Frame(number, // Frame number
94  * 720, // Width of image
95  * 480, // Height of image
96  * "#000000" // HTML color code of background color
97  * 44100, // Sample rate of audio stream
98  * 2 // Number of audio channels
99  * );
100  *
101  * // Some methods require a shared pointer to an openshot::Frame object.
102  * std::shared_ptr<Frame> f(new Frame(1, 720, 480, "#000000", 44100, 2));
103  *
104  * @endcode
105  */
106  class Frame
107  {
108  private:
109  std::shared_ptr<QImage> image;
110  std::shared_ptr<QImage> wave_image;
111  std::shared_ptr<juce::AudioSampleBuffer> audio;
112  std::shared_ptr<QApplication> previewApp;
113  juce::CriticalSection addingImageSection;
114  juce::CriticalSection addingAudioSection;
115  const unsigned char *qbuffer;
116  openshot::Fraction pixel_ratio;
117  int channels;
118  ChannelLayout channel_layout;
119  int width;
120  int height;
121  int sample_rate;
122  std::string color;
123  int64_t max_audio_sample; ///< The max audio sample count added to this frame
124 
125  /// Constrain a color value from 0 to 255
126  int constrain(int color_value);
127 
128  public:
129  int64_t number; ///< This is the frame number (starting at 1)
130  bool has_audio_data; ///< This frame has been loaded with audio data
131  bool has_image_data; ///< This frame has been loaded with pixel data
132 
133 
134  /// Constructor - blank frame (300x200 blank image, 48kHz audio silence)
135  Frame();
136 
137  /// Constructor - image only (48kHz audio silence)
138  Frame(int64_t number, int width, int height, std::string color);
139 
140  /// Constructor - audio only (300x200 blank image)
141  Frame(int64_t number, int samples, int channels);
142 
143  /// Constructor - image & audio
144  Frame(int64_t number, int width, int height, std::string color, int samples, int channels);
145 
146  /// Copy constructor
147  Frame ( const Frame &other );
148 
149  /// Assignment operator
150  Frame& operator= (const Frame& other);
151 
152  /// Destructor
153  virtual ~Frame();
154 
155  /// Add (or replace) pixel data to the frame (based on a solid color)
156  void AddColor(int new_width, int new_height, std::string new_color);
157 
158  /// Add (or replace) pixel data to the frame
159  void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_);
160 
161  /// Add (or replace) pixel data to the frame
162  void AddImage(std::shared_ptr<QImage> new_image);
163 
164  /// Add (or replace) pixel data to the frame (for only the odd or even lines)
165  void AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines);
166 
167 #ifdef USE_IMAGEMAGICK
168  /// Add (or replace) pixel data to the frame from an ImageMagick Image
169  void AddMagickImage(std::shared_ptr<Magick::Image> new_image);
170 #endif
171 
172  /// Add audio samples to a specific channel
173  void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource);
174 
175  /// Add audio silence
176  void AddAudioSilence(int numSamples);
177 
178  /// Apply gain ramp (i.e. fading volume)
179  void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain);
180 
181  /// Channel Layout of audio samples. A frame needs to keep track of this, since Writers do not always
182  /// know the original channel layout of a frame's audio samples (i.e. mono, stereo, 5 point surround, etc...)
184 
185  // Set the channel layout of audio samples (i.e. mono, stereo, 5 point surround, etc...)
186  void ChannelsLayout(openshot::ChannelLayout new_channel_layout) { channel_layout = new_channel_layout; };
187 
188  /// Clean up buffer after QImage is deleted
189  static void cleanUpBuffer(void *info);
190 
191  /// Clear the waveform image (and deallocate its memory)
192  void ClearWaveform();
193 
194  /// Copy data and pointers from another Frame instance
195  void DeepCopy(const Frame& other);
196 
197  /// Display the frame image to the screen (primarily used for debugging reasons)
198  void Display();
199 
200  /// Display the wave form
201  void DisplayWaveform();
202 
203  /// Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
204  float GetAudioSample(int channel, int sample, int magnitude_range);
205 
206  /// Get an array of sample data
207  float* GetAudioSamples(int channel);
208 
209  /// Get an array of sample data (all channels interleaved together), using any sample rate
210  float* GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler* resampler, int* sample_count);
211 
212  // Get a planar array of sample data, using any sample rate
213  float* GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler* resampler, int* sample_count);
214 
215  /// Get number of audio channels
216  int GetAudioChannelsCount();
217 
218  /// Get number of audio samples
219  int GetAudioSamplesCount();
220 
221  juce::AudioSampleBuffer *GetAudioSampleBuffer();
222 
223  /// Get the size in bytes of this frame (rough estimate)
224  int64_t GetBytes();
225 
226  /// Get pointer to Qt QImage image object
227  std::shared_ptr<QImage> GetImage();
228 
229 #ifdef USE_IMAGEMAGICK
230  /// Get pointer to ImageMagick image object
231  std::shared_ptr<Magick::Image> GetMagickImage();
232 #endif
233 
234  /// Set Pixel Aspect Ratio
235  openshot::Fraction GetPixelRatio() { return pixel_ratio; };
236 
237  /// Get pixel data (as packets)
238  const unsigned char* GetPixels();
239 
240  /// Get pixel data (for only a single scan-line)
241  const unsigned char* GetPixels(int row);
242 
243  /// Check a specific pixel color value (returns True/False)
244  bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold);
245 
246  /// Get height of image
247  int GetHeight();
248 
249  /// Calculate the # of samples per video frame (for the current frame number)
250  int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels);
251 
252  /// Calculate the # of samples per video frame (for a specific frame number and frame rate)
253  static int GetSamplesPerFrame(int64_t frame_number, openshot::Fraction fps, int sample_rate, int channels);
254 
255  /// Get an audio waveform image
256  std::shared_ptr<QImage> GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha);
257 
258  /// Get an audio waveform image pixels
259  const unsigned char* GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha);
260 
261  /// Get height of image
262  int GetWidth();
263 
264  /// Resize audio container to hold more (or less) samples and channels
265  void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout);
266 
267  /// Get the original sample rate of this frame's audio data
268  int SampleRate();
269 
270  /// Set the original sample rate of this frame's audio data
271  void SampleRate(int orig_sample_rate) { sample_rate = orig_sample_rate; };
272 
273  /// Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM
274  void Save(std::string path, float scale, std::string format="PNG", int quality=100);
275 
276  /// Set frame number
277  void SetFrameNumber(int64_t number);
278 
279  /// Set Pixel Aspect Ratio
280  void SetPixelRatio(int num, int den);
281 
282  /// Thumbnail the frame image with tons of options to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG).
283  /// This method allows for masks, overlays, background color, and much more accurate resizing (including padding and centering)
284  void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
285  std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0);
286 
287  /// Play audio samples for this frame
288  void Play();
289  };
290 
291 }
292 
293 #endif
openshot::Fraction GetPixelRatio()
Set Pixel Aspect Ratio.
Definition: Frame.h:235
void AddMagickImage(std::shared_ptr< Magick::Image > new_image)
Add (or replace) pixel data to the frame from an ImageMagick Image.
Definition: Frame.cpp:949
int GetWidth()
Get height of image.
Definition: Frame.cpp:559
Header file for Fraction class.
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:442
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:733
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:609
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:860
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:106
std::shared_ptr< Magick::Image > GetMagickImage()
Get pointer to ImageMagick image object.
Definition: Frame.cpp:925
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:278
Header file for MagickUtilities (IM6/IM7 compatibility overlay)
juce::AudioSampleBuffer * GetAudioSampleBuffer()
Definition: Frame.cpp:448
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:469
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:975
Header file for AudioBufferSource class.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:105
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:874
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:135
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:129
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:96
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:547
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:271
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:341
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:334
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:517
Header file for AudioResampler class.
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:1056
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:130
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
Definition: Frame.cpp:387
This class represents a fraction.
Definition: Fraction.h:45
Header file for ZeroMQ-based Logger class.
Header file for ChannelLayout class.
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
Definition: Frame.cpp:1045
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
void ChannelsLayout(openshot::ChannelLayout new_channel_layout)
Definition: Frame.h:186
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:754
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
Definition: Frame.cpp:37
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:904
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:288
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:510
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:432
This namespace is the default namespace for all code in the openshot library.
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:571
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:913
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:176
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:454
virtual ~Frame()
Destructor.
Definition: Frame.cpp:128
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
Definition: Frame.cpp:321
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:131
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
Definition: Frame.cpp:578
int GetHeight()
Get height of image.
Definition: Frame.cpp:553
void SampleRate(int orig_sample_rate)
Set the original sample rate of this frame&#39;s audio data.
Definition: Frame.h:271
int SampleRate()
Get the original sample rate of this frame&#39;s audio data.
Definition: Frame.cpp:565
This class is used to resample audio data for many sequential frames.
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:488